jekyll-import 0.1.0.beta3 → 0.1.0.beta4

data/lib/jekyll-import/importers/marley.rb

@@ -0,0 +1,72 @@
+module JekyllImport
+  module Importers
+    class Marley < Importer
+      def self.validate(options)
+        if options['marley_data_dir'].nil?
+          Jekyll.logger.abort_with "Missing mandatory option --marley_data_dir."
+        else
+          unless File.directory?(options['marley_data_dir'])
+            raise ArgumentError, "marley dir '#{options['marley_data_dir']}' not found"
+          end
+        end
+      end
+
+      def self.regexp
+        { :id    => /^\d{0,4}-{0,1}(.*)$/,
+          :title => /^#\s*(.*)\s+$/,
+          :title_with_date => /^#\s*(.*)\s+\(([0-9\/]+)\)$/,
+          :published_on => /.*\s+\(([0-9\/]+)\)$/,
+          :perex => /^([^\#\n]+\n)$/,
+          :meta  => /^\{\{\n(.*)\}\}\n$/mi # Multiline Regexp
+        }
+      end
+
+      def self.require_deps
+        JekyllImport.require_with_fallback(%w[
+          fileutils
+          safe_yaml
+        ])
+      end
+
+      def self.specify_options(c)
+        c.option 'marley_data_dir', '--marley_data_dir DIR', 'The dir containing your marley data'
+      end
+
+      def self.process(options)
+        marley_data_dir = options.fetch('marley_data_dir')
+
+        FileUtils.mkdir_p "_posts"
+
+        posts = 0
+        Dir["#{marley_data_dir}/**/*.txt"].each do |f|
+          next unless File.exists?(f)
+
+          #copied over from marley's app/lib/post.rb
+          file_content  = File.read(f)
+          meta_content  = file_content.slice!( self.regexp[:meta] )
+          body          = file_content.sub( self.regexp[:title], '').sub( self.regexp[:perex], '').strip
+
+          title = file_content.scan( self.regexp[:title] ).first.to_s.strip
+          prerex = file_content.scan( self.regexp[:perex] ).first.to_s.strip
+          published_on = DateTime.parse( post[:published_on] ) rescue File.mtime( File.dirname(f) )
+          meta          = ( meta_content ) ? YAML::load( meta_content.scan( self.regexp[:meta]).to_s ) : {}
+          meta['title'] = title
+          meta['layout'] = 'post'
+
+          formatted_date = published_on.strftime('%Y-%m-%d')
+          post_name =  File.dirname(f).split(%r{/}).last.gsub(/\A\d+-/, '')
+
+          name = "#{formatted_date}-#{post_name}"
+          File.open("_posts/#{name}.markdown", "w") do |f|
+            f.puts meta.to_yaml
+            f.puts "---\n"
+            f.puts "\n#{prerex}\n\n" if prerex
+            f.puts body
+          end
+          posts += 1
+        end
+        "Created #{posts} posts!"
+      end
+    end
+  end
+end

data/lib/jekyll-import/importers/mephisto.rb

@@ -0,0 +1,109 @@
+# Quickly hacked together my Michael Ivey
+# Based on mt.rb by Nick Gerakines, open source and publically
+# available under the MIT license. Use this module at your own risk.
+
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module JekyllImport
+  module Importers
+    class Mephisto < Importer
+      #Accepts a hash with database config variables, exports mephisto posts into a csv
+      #export PGPASSWORD if you must
+      def self.postgres(c)
+        sql = <<-SQL
+        BEGIN;
+        CREATE TEMP TABLE jekyll AS
+          SELECT title, permalink, body, published_at, filter FROM contents
+          WHERE user_id = 1 AND type = 'Article' ORDER BY published_at;
+        COPY jekyll TO STDOUT WITH CSV HEADER;
+        ROLLBACK;
+        SQL
+        command = %Q(psql -h #{c[:host] || "localhost"} -c "#{sql.strip}" #{c[:database]} #{c[:username]} -o #{c[:filename] || "posts.csv"})
+        puts command
+        `#{command}`
+        CSV.process
+      end
+
+      def self.validate(options)
+        %w[dbname user].each do |option|
+          if options[option].nil?
+            abort "Missing mandatory option --#{option}."
+          end
+        end
+      end
+
+      def self.require_deps
+        JekyllImport.require_with_fallback(%w[
+          rubygems
+          sequel
+          fastercsv
+          fileutils
+        ])
+      end
+
+      def self.specify_options(c)
+        c.option 'dbname', '--dbname DB', 'Database name'
+        c.option 'user', '--user USER', 'Database user name'
+        c.option 'password', '--password PW', "Database user's password (default: '')"
+        c.option 'host', '--host HOST', 'Database host name (default: "localhost")'
+      end
+
+      # This query will pull blog posts from all entries across all blogs. If
+      # you've got unpublished, deleted or otherwise hidden posts please sift
+      # through the created posts to make sure nothing is accidently published.
+      QUERY = "SELECT id, \
+                      permalink, \
+                      body, \
+                      published_at, \
+                      title \
+               FROM contents \
+               WHERE user_id = 1 AND \
+                     type = 'Article' AND \
+                     published_at IS NOT NULL \
+               ORDER BY published_at"
+
+      def self.process(options)
+        dbname = options.fetch('dbname')
+        user   = options.fetch('user')
+        pass   = options.fetch('password', '')
+        host   = options.fetch('host', "localhost")
+
+        db = Sequel.mysql(dbname, :user => user,
+                                  :password => pass,
+                                  :host => host,
+                                  :encoding => 'utf8')
+
+        FileUtils.mkdir_p "_posts"
+
+        db[QUERY].each do |post|
+          title = post[:title]
+          slug = post[:permalink]
+          date = post[:published_at]
+          content = post[:body]
+
+          # Ideally, this script would determine the post format (markdown,
+          # html, etc) and create files with proper extensions. At this point
+          # it just assumes that markdown will be acceptable.
+          name = [date.year, date.month, date.day, slug].join('-') + ".markdown"
+
+          data = {
+             'layout' => 'post',
+             'title' => title.to_s,
+             'mt_id' => post[:entry_id],
+           }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+
+          File.open("_posts/#{name}", "w") do |f|
+            f.puts data
+            f.puts "---"
+            f.puts content
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/jekyll-import/importers/mt.rb

@@ -0,0 +1,169 @@
+# Created by Nick Gerakines, open source and publically available under the
+# MIT license. Use this module at your own risk.
+# I'm an Erlang/Perl/C++ guy so please forgive my dirty ruby.
+
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module JekyllImport
+  module Importers
+    class MT < Importer
+
+      STATUS_DRAFT = 1
+      STATUS_PUBLISHED = 2
+      MORE_CONTENT_SEPARATOR = '<!--more-->'
+
+      def self.default_options
+        {
+          :blog_id => nil,
+          :categories => true,
+          :dest_encoding => 'utf-8',
+          :src_encoding => 'utf-8'
+        }
+      end
+
+      def self.require_deps
+        JekyllImport.require_with_fallback(%w[
+          rubygems
+          sequel
+          fileutils
+          safe_yaml
+        ])
+      end
+
+      def self.specify_options(c)
+        c.option 'dbname', '--dbname DB', 'Database name'
+        c.option 'user', '--user USER', 'Database user name'
+        c.option 'password', '--password PW', "Database user's password, (default: '')"
+        c.option 'host', '--host HOST', 'Database host name (default: "localhost")'
+      end
+
+      # By default this migrator will include posts for all your MovableType blogs.
+      # Specify a single blog by providing blog_id.
+
+      # Main migrator function. Call this to perform the migration.
+      #
+      # dbname::  The name of the database
+      # user::    The database user name
+      # pass::    The database user's password
+      # host::    The address of the MySQL database host. Default: 'localhost'
+      # options:: A hash of configuration options
+      #
+      # Supported options are:
+      #
+      # :blog_id::        Specify a single MovableType blog to export by providing blog_id.
+      #                   Default: nil, importer will include posts for all blogs.
+      # :categories::     If true, save the post's categories in its
+      #                   YAML front matter. Default: true
+      # :src_encoding::   Encoding of strings from the database. Default: UTF-8
+      #                   If your output contains mangled characters, set src_encoding to
+      #                   something appropriate for your database charset.
+      # :dest_encoding::  Encoding of output strings. Default: UTF-8
+      def self.process(options)
+        dbname = options.fetch('dbname')
+        user   = options.fetch('user')
+        pass   = options.fetch('password', "")
+        host   = options.fetch('host', "localhost")
+
+        options = default_options.merge(options)
+
+        db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host)
+        post_categories = db[:mt_placement].join(:mt_category, :category_id => :placement_category_id)
+
+        FileUtils.mkdir_p "_posts"
+
+        posts = db[:mt_entry]
+        posts = posts.filter(:entry_blog_id => options[:blog_id]) if options[:blog_id]
+        posts.each do |post|
+          categories = post_categories.filter(
+            :mt_placement__placement_entry_id => post[:entry_id]
+          ).map {|ea| encode(ea[:category_basename], options) }
+
+          file_name = post_file_name(post, options)
+
+          data = post_metadata(post, options)
+          data['categories'] = categories if !categories.empty? && options[:categories]
+          yaml_front_matter = data.delete_if { |k,v| v.nil? || v == '' }.to_yaml
+
+          content = post_content(post, options)
+
+          File.open("_posts/#{file_name}", "w") do |f|
+            f.puts yaml_front_matter
+            f.puts "---"
+            f.puts encode(content, options)
+          end
+        end
+      end
+
+      # Extracts metadata for YAML front matter from post
+      def self.post_metadata(post, options = default_options)
+        metadata = {
+          'layout' => 'post',
+          'title' => encode(post[:entry_title], options),
+          'date' => post_date(post).strftime("%Y-%m-%d %H:%M:%S %z"),
+          'excerpt' => encode(post[:entry_excerpt], options),
+          'mt_id' => post[:entry_id]
+        }
+        metadata['published'] = false if post[:entry_status] != STATUS_PUBLISHED
+        metadata
+      end
+
+      # Different versions of MT used different column names
+      def self.post_date(post)
+        post[:entry_authored_on] || post[:entry_created_on]
+      end
+
+      # Extracts text body from post
+      def self.extra_entry_text_empty?(post)
+        post[:entry_text_more].nil? || post[:entry_text_more].strip.empty?
+      end
+
+      def self.post_content(post, options = default_options)
+        if extra_entry_text_empty?(post)
+          post[:entry_text]
+        else
+          post[:entry_text] + "\n\n#{MORE_CONTENT_SEPARATOR}\n\n" + post[:entry_text_more]
+        end
+      end
+
+      def self.post_file_name(post, options = default_options)
+        date = post_date(post)
+        slug = post[:entry_basename]
+        file_ext = suffix(post[:entry_convert_breaks])
+
+        "#{date.strftime('%Y-%m-%d')}-#{slug}.#{file_ext}"
+      end
+
+      def self.encode(str, options = default_options)
+        if str.respond_to?(:encoding)
+          str.encode(options[:dest_encoding], options[:src_encoding])
+        else
+          str
+        end
+      end
+
+      # Ideally, this script would determine the post format (markdown,
+      # html, etc) and create files with proper extensions. At this point
+      # it just assumes that markdown will be acceptable.
+      def self.suffix(entry_type)
+        if entry_type.nil? || entry_type.include?("markdown") || entry_type.include?("__default__")
+          # The markdown plugin I have saves this as
+          # "markdown_with_smarty_pants", so I just look for "markdown".
+          "markdown"
+        elsif entry_type.include?("textile")
+          # This is saved as "textile_2" on my installation of MT 5.1.
+          "textile"
+        elsif entry_type == "0" || entry_type.include?("richtext")
+          # Richtext looks to me like it's saved as HTML, so I include it here.
+          "html"
+        else
+          # Other values might need custom work.
+          entry_type
+        end
+      end
+    end
+  end
+end

data/lib/jekyll-import/importers/posterous.rb

@@ -0,0 +1,139 @@
+# ruby -r './lib/jekyll/migrators/posterous.rb' -e 'Jekyll::Posterous.process(email, pass, api_key)'
+# Other arguments are optional; the default values are:
+# * :include_imgs => false         # should images be downloaded as well?
+# * :blog         => 'primary'     # blog, if you have more than one.
+# * :base_path    => '/'           # for image, if they will be served from a different host for eg.
+
+# For example, to download images as well as your posts, use the above command with
+# ....process(email, pass, api_key, :include_imgs => true)
+
+module JekyllImport
+  module Importers
+    class Posterous < Importer
+
+      def self.specify_options(c)
+        c.option 'email', '--email EMAIL', 'Posterous email address'
+        c.option 'password', '--password PW', 'Posterous password'
+        c.option 'api_token', '--token TOKEN', 'Posterous API Token'
+      end
+
+      def self.require_deps
+        JekyllImport.require_with_fallback(%w[
+          rubygems
+          jekyll
+          fileutils
+          uri
+          json
+          net/http
+        ])
+      end
+
+      def self.fetch(uri_str, limit = 10)
+        # You should choose better exception.
+        raise ArgumentError, 'Stuck in a redirect loop. Please double check your email and password' if limit == 0
+
+        response = nil
+        Net::HTTP.start('posterous.com') do |http|
+          req = Net::HTTP::Get.new(uri_str)
+          req.basic_auth @email, @pass
+          response = http.request(req)
+        end
+
+        case response
+          when Net::HTTPSuccess     then response
+          when Net::HTTPRedirection then fetch(response['location'], limit - 1)
+          else response.error!
+        end
+      end
+
+      def self.fetch_images(directory, imgs)
+        def self.fetch_one(url, limit = 10)
+          raise ArgumentError, 'HTTP redirect too deep' if limit == 0
+          response = Net::HTTP.get_response(URI.parse(url))
+          case response
+          when Net::HTTPSuccess     then response.body
+          when Net::HTTPRedirection then self.fetch_one(response['location'], limit - 1)
+          else
+            response.error!
+          end
+        end
+
+        FileUtils.mkdir_p directory
+        urls = Array.new
+        imgs.each do |img|
+          fullurl = img["full"]["url"]
+          uri = URI.parse(fullurl)
+          imgname = uri.path.split("/")[-1]
+          imgdata = self.fetch_one(fullurl)
+          open(directory + "/" + imgname, "wb") do |file|
+            file.write imgdata
+          end
+          urls.push(directory + "/" + imgname)
+        end
+
+        return urls
+      end
+
+      def self.process(options)
+        email     = options.fetch('email')
+        pass      = options.fetch('password')
+        api_token = options.fetch('api_token')
+
+        @email, @pass, @api_token = email, pass, api_token
+        defaults = { :include_imgs => false, :blog => 'primary', :base_path => '/' }
+        opts = defaults.merge(opts)
+        FileUtils.mkdir_p "_posts"
+
+        posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{opts[:blog]}/posts?api_token=#{@api_token}").body)
+        page = 1
+
+        while posts.any?
+          posts.each do |post|
+            title = post["title"]
+            slug = title.gsub(/[^[:alnum:]]+/, '-').downcase
+            date = Date.parse(post["display_date"])
+            content = post["body_html"]
+            published = !post["is_private"]
+            basename = "%02d-%02d-%02d-%s" % [date.year, date.month, date.day, slug]
+            name = basename + '.html'
+
+            # Images:
+            if opts[:include_imgs]
+              post_imgs = post["media"]["images"]
+              if post_imgs.any?
+                img_dir = "imgs/%s" % basename
+                img_urls = self.fetch_images(img_dir, post_imgs)
+
+                img_urls.map! do |url|
+                  '<li><img src="' + opts[:base_path] + url + '"></li>'
+                end
+                imgcontent = "<ol>\n" + img_urls.join("\n") + "</ol>\n"
+
+                # filter out "posterous-content", replacing with imgs:
+                content = content.sub(/\<p\>\[\[posterous-content:[^\]]+\]\]\<\/p\>/, imgcontent)
+              end
+            end
+
+            # Get the relevant fields as a hash, delete empty fields and convert
+            # to YAML for the header
+            data = {
+               'layout' => 'post',
+               'title' => title.to_s,
+               'published' => published
+             }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+
+            # Write out the data and content to file
+            File.open("_posts/#{name}", "w") do |f|
+              f.puts data
+              f.puts "---"
+              f.puts content
+            end
+          end
+
+          page += 1
+          posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{opts[:blog]}/posts?api_token=#{@api_token}&page=#{page}").body)
+        end
+      end
+    end
+  end
+end