jekyll-import 0.1.0.beta1

data/lib/jekyll/jekyll-import/drupal7.rb

@@ -0,0 +1,73 @@
+require 'rubygems'
+require 'sequel'
+require 'fileutils'
+require 'yaml'
+
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module JekyllImport
+  module Drupal7
+    # Reads a MySQL database via Sequel and creates a post file for each post
+    # in wp_posts that has post_status = 'publish'. This restriction is made
+    # because 'draft' posts are not guaranteed to have valid dates.
+    QUERY = "SELECT n.nid, \
+                    n.title, \
+                    fdb.body_value, \
+                    n.created, \
+                    n.status \
+             FROM node AS n, \
+                  field_data_body AS fdb \
+             WHERE (n.type = 'blog' OR n.type = 'story') \
+             AND n.vid = fdb.entity_id"
+
+    def self.process(dbname, user, pass, host = 'localhost', prefix = '')
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+
+      if prefix != ''
+        QUERY[" node "] = " " + prefix + "node "
+        QUERY[" field_data_body "] = " " + prefix + "field_data_body "
+      end
+
+      FileUtils.mkdir_p "_posts"
+      FileUtils.mkdir_p "_drafts"
+
+      db[QUERY].each do |post|
+        # Get required fields and construct Jekyll compatible name
+        node_id = post[:nid]
+        title = post[:title]
+        content = post[:body_value]
+        created = post[:created]
+        time = Time.at(created)
+        is_published = post[:status] == 1
+        dir = is_published ? "_posts" : "_drafts"
+        slug = title.strip.downcase.gsub(/(&|&)/, ' and ').gsub(/[\s\.\/\\]/, '-').gsub(/[^\w-]/, '').gsub(/[-_]{2,}/, '-').gsub(/^[-_]/, '').gsub(/[-_]$/, '')
+        name = time.strftime("%Y-%m-%d-") + slug + '.md'
+
+        # Get the relevant fields as a hash, delete empty fields and convert
+        # to YAML for the header
+        data = {
+           'layout' => 'default',
+           'title' => title.to_s,
+           'created' => created,
+         }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+
+        # Write out the data and content to file
+        File.open("#{dir}/#{name}", "w") do |f|
+          f.puts data
+          f.puts "---"
+          f.puts content
+        end
+
+      end
+
+      # TODO: Make dirs & files for nodes of type 'page'
+        # Make refresh pages for these as well
+
+      # TODO: Make refresh dirs & files according to entries in url_alias table
+    end
+  end
+end

data/lib/jekyll/jekyll-import/enki.rb

@@ -0,0 +1,49 @@
+# Adapted by Rodrigo Pinto <rodrigopqn@gmail.com>
+# Based on typo.rb by Toby DiPasquale
+
+require 'fileutils'
+require 'rubygems'
+require 'sequel'
+
+module JekyllImport
+  module Enki
+    SQL = <<-EOS
+      SELECT p.id,
+             p.title,
+             p.slug,
+             p.body,
+             p.published_at as date,
+             p.cached_tag_list as tags
+      FROM posts p
+    EOS
+
+    # Just working with postgres, but can be easily adapted
+    # to work with both mysql and postgres.
+    def self.process(dbname, user, pass, host = 'localhost')
+      FileUtils.mkdir_p('_posts')
+      db = Sequel.postgres(:database => dbname,
+                           :user => user,
+                           :password => pass,
+                           :host => host,
+                           :encoding => 'utf8')
+
+      db[SQL].each do |post|
+        name = [ sprintf("%.04d", post[:date].year),
+                 sprintf("%.02d", post[:date].month),
+                 sprintf("%.02d", post[:date].day),
+                 post[:slug].strip ].join('-')
+        name += '.textile'
+
+        File.open("_posts/#{name}", 'w') do |f|
+          f.puts({ 'layout'   => 'post',
+                   'title'    => post[:title].to_s,
+                   'enki_id'  => post[:id],
+                   'categories'  => post[:tags]
+                 }.delete_if { |k, v| v.nil? || v == '' }.to_yaml)
+          f.puts '---'
+          f.puts post[:body].delete("\r")
+        end
+      end
+    end
+  end
+end

data/lib/jekyll/jekyll-import/joomla.rb

@@ -0,0 +1,53 @@
+require 'rubygems'
+require 'sequel'
+require 'fileutils'
+require 'safe_yaml'
+
+# NOTE: This migrator is made for Joomla 1.5 databases.
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module JekyllImport
+  module Joomla
+    def self.process(dbname, user, pass, host = 'localhost', table_prefix = 'jos_', section = '1')
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+
+      FileUtils.mkdir_p("_posts")
+
+      # Reads a MySQL database via Sequel and creates a post file for each
+      # post in wp_posts that has post_status = 'publish'. This restriction is
+      # made because 'draft' posts are not guaranteed to have valid dates.
+      query = "SELECT `title`, `alias`, CONCAT(`introtext`,`fulltext`) as content, `created`, `id` FROM #{table_prefix}content WHERE state = '0' OR state = '1' AND sectionid = '#{section}'"
+
+      db[query].each do |post|
+        # Get required fields and construct Jekyll compatible name.
+        title = post[:title]
+        slug = post[:alias]
+        date = post[:created]
+        content = post[:content]
+        name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day,
+                                               slug]
+
+        # Get the relevant fields as a hash, delete empty fields and convert
+        # to YAML for the header.
+        data = {
+           'layout' => 'post',
+           'title' => title.to_s,
+           'joomla_id' => post[:id],
+           'joomla_url' => post[:alias],
+           'date' => date
+         }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
+
+        # Write out the data and content to file
+        File.open("_posts/#{name}", "w") do |f|
+          f.puts data
+          f.puts "---"
+          f.puts content
+        end
+      end
+    end
+  end
+end

data/lib/jekyll/jekyll-import/marley.rb

@@ -0,0 +1,52 @@
+require 'safe_yaml'
+require 'fileutils'
+
+module JekyllImport
+  module Marley
+    def self.regexp
+      { :id    => /^\d{0,4}-{0,1}(.*)$/,
+        :title => /^#\s*(.*)\s+$/,
+        :title_with_date => /^#\s*(.*)\s+\(([0-9\/]+)\)$/,
+        :published_on => /.*\s+\(([0-9\/]+)\)$/,
+        :perex => /^([^\#\n]+\n)$/,
+        :meta  => /^\{\{\n(.*)\}\}\n$/mi # Multiline Regexp 
+      }
+    end
+
+    def self.process(marley_data_dir)
+      raise ArgumentError, "marley dir #{marley_data_dir} not found" unless File.directory?(marley_data_dir)
+
+      FileUtils.mkdir_p "_posts"
+
+      posts = 0
+      Dir["#{marley_data_dir}/**/*.txt"].each do |f|
+        next unless File.exists?(f)
+
+        #copied over from marley's app/lib/post.rb
+        file_content  = File.read(f)
+        meta_content  = file_content.slice!( self.regexp[:meta] )
+        body          = file_content.sub( self.regexp[:title], '').sub( self.regexp[:perex], '').strip
+   
+        title = file_content.scan( self.regexp[:title] ).first.to_s.strip
+        prerex = file_content.scan( self.regexp[:perex] ).first.to_s.strip
+        published_on = DateTime.parse( post[:published_on] ) rescue File.mtime( File.dirname(f) )
+        meta          = ( meta_content ) ? YAML::load( meta_content.scan( self.regexp[:meta]).to_s ) : {}
+        meta['title'] = title
+        meta['layout'] = 'post'
+
+        formatted_date = published_on.strftime('%Y-%m-%d')
+        post_name =  File.dirname(f).split(%r{/}).last.gsub(/\A\d+-/, '')
+
+        name = "#{formatted_date}-#{post_name}" 
+        File.open("_posts/#{name}.markdown", "w") do |f|
+          f.puts meta.to_yaml
+          f.puts "---\n"
+          f.puts "\n#{prerex}\n\n" if prerex
+          f.puts body
+        end
+        posts += 1
+      end
+      "Created #{posts} posts!"
+    end
+  end
+end

data/lib/jekyll/jekyll-import/mephisto.rb

@@ -0,0 +1,84 @@
+# Quickly hacked together my Michael Ivey
+# Based on mt.rb by Nick Gerakines, open source and publically
+# available under the MIT license. Use this module at your own risk.
+
+require 'rubygems'
+require 'sequel'
+require 'fastercsv'
+require 'fileutils'
+require File.join(File.dirname(__FILE__),"csv.rb")
+
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module JekyllImport
+  module Mephisto
+    #Accepts a hash with database config variables, exports mephisto posts into a csv
+    #export PGPASSWORD if you must
+    def self.postgres(c)
+      sql = <<-SQL
+      BEGIN;
+      CREATE TEMP TABLE jekyll AS
+        SELECT title, permalink, body, published_at, filter FROM contents
+        WHERE user_id = 1 AND type = 'Article' ORDER BY published_at;
+      COPY jekyll TO STDOUT WITH CSV HEADER;
+      ROLLBACK;
+      SQL
+      command = %Q(psql -h #{c[:host] || "localhost"} -c "#{sql.strip}" #{c[:database]} #{c[:username]} -o #{c[:filename] || "posts.csv"})
+      puts command
+      `#{command}`
+      CSV.process
+    end
+
+    # This query will pull blog posts from all entries across all blogs. If
+    # you've got unpublished, deleted or otherwise hidden posts please sift
+    # through the created posts to make sure nothing is accidently published.
+    QUERY = "SELECT id, \
+                    permalink, \
+                    body, \
+                    published_at, \
+                    title \
+             FROM contents \
+             WHERE user_id = 1 AND \
+                   type = 'Article' AND \
+                   published_at IS NOT NULL \
+             ORDER BY published_at"
+
+    def self.process(dbname, user, pass, host = 'localhost')
+      db = Sequel.mysql(dbname, :user => user,
+                                :password => pass,
+                                :host => host,
+                                :encoding => 'utf8')
+
+      FileUtils.mkdir_p "_posts"
+
+      db[QUERY].each do |post|
+        title = post[:title]
+        slug = post[:permalink]
+        date = post[:published_at]
+        content = post[:body]
+
+        # Ideally, this script would determine the post format (markdown,
+        # html, etc) and create files with proper extensions. At this point
+        # it just assumes that markdown will be acceptable.
+        name = [date.year, date.month, date.day, slug].join('-') + ".markdown"
+
+        data = {
+           'layout' => 'post',
+           'title' => title.to_s,
+           'mt_id' => post[:entry_id],
+         }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+
+        File.open("_posts/#{name}", "w") do |f|
+          f.puts data
+          f.puts "---"
+          f.puts content
+        end
+      end
+
+    end
+  end
+end

data/lib/jekyll/jekyll-import/mt.rb

@@ -0,0 +1,142 @@
+# Created by Nick Gerakines, open source and publically available under the
+# MIT license. Use this module at your own risk.
+# I'm an Erlang/Perl/C++ guy so please forgive my dirty ruby.
+
+require 'rubygems'
+require 'sequel'
+require 'fileutils'
+require 'safe_yaml'
+
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module JekyllImport
+  module MT
+
+    STATUS_DRAFT = 1
+    STATUS_PUBLISHED = 2
+    MORE_CONTENT_SEPARATOR = '<!--more-->'
+
+    def self.default_options
+      {
+        :blog_id => nil,
+        :categories => true,
+        :dest_encoding => 'utf-8',
+        :src_encoding => 'utf-8'
+      }
+    end
+
+    # By default this migrator will include posts for all your MovableType blogs.
+    # Specify a single blog by providing blog_id.
+
+    # Main migrator function. Call this to perform the migration.
+    #
+    # dbname::  The name of the database
+    # user::    The database user name
+    # pass::    The database user's password
+    # host::    The address of the MySQL database host. Default: 'localhost'
+    # options:: A hash of configuration options
+    #
+    # Supported options are:
+    #
+    # :blog_id::        Specify a single MovableType blog to export by providing blog_id.
+    #                   Default: nil, importer will include posts for all blogs.
+    # :categories::     If true, save the post's categories in its
+    #                   YAML front matter. Default: true
+    # :src_encoding::   Encoding of strings from the database. Default: UTF-8
+    #                   If your output contains mangled characters, set src_encoding to
+    #                   something appropriate for your database charset.
+    # :dest_encoding::  Encoding of output strings. Default: UTF-8
+    def self.process(dbname, user, pass, host = 'localhost', options = {})
+      options = default_options.merge(options)
+
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host)
+      post_categories = db[:mt_placement].join(:mt_category, :category_id => :placement_category_id)
+
+      FileUtils.mkdir_p "_posts"
+
+      posts = db[:mt_entry]
+      posts = posts.filter(:entry_blog_id => options[:blog_id]) if options[:blog_id]
+      posts.each do |post|
+        categories = post_categories.filter(
+          :mt_placement__placement_entry_id => post[:entry_id]
+        ).map {|ea| encode(ea[:category_basename], options) }
+
+        file_name = post_file_name(post, options)
+
+        data = post_metadata(post, options)
+        data['categories'] = categories if !categories.empty? && options[:categories]
+        yaml_front_matter = data.delete_if { |k,v| v.nil? || v == '' }.to_yaml
+
+        content = post_content(post, options)
+
+        File.open("_posts/#{file_name}", "w") do |f|
+          f.puts yaml_front_matter
+          f.puts "---"
+          f.puts encode(content, options)
+        end
+      end
+    end
+
+    # Extracts metadata for YAML front matter from post
+    def self.post_metadata(post, options = default_options)
+      metadata = {
+        'layout' => 'post',
+        'title' => encode(post[:entry_title], options),
+        'date' => post[:entry_authored_on].strftime("%Y-%m-%d %H:%M:%S %z"),
+        'excerpt' => encode(post[:entry_excerpt], options),
+        'mt_id' => post[:entry_id]
+      }
+      metadata['published'] = false if post[:entry_status] != STATUS_PUBLISHED
+      metadata
+    end
+
+    # Extracts text body from post
+    def self.post_content(post, options = default_options)
+      if post[:entry_text_more].strip.empty?
+        post[:entry_text]
+      else
+        post[:entry_text] + "\n\n#{MORE_CONTENT_SEPARATOR}\n\n" + post[:entry_text_more]
+      end
+    end
+
+    def self.post_file_name(post, options = default_options)
+      date = post[:entry_authored_on]
+      slug = post[:entry_basename]
+      file_ext = suffix(post[:entry_convert_breaks])
+
+      "#{date.strftime('%Y-%m-%d')}-#{slug}.#{file_ext}"
+    end
+
+    def self.encode(str, options = default_options)
+      if str.respond_to?(:encoding)
+        str.encode(options[:dest_encoding], options[:src_encoding])
+      else
+        str
+      end
+    end
+
+    # Ideally, this script would determine the post format (markdown,
+    # html, etc) and create files with proper extensions. At this point
+    # it just assumes that markdown will be acceptable.
+    def self.suffix(entry_type)
+      if entry_type.nil? || entry_type.include?("markdown") || entry_type.include?("__default__")
+        # The markdown plugin I have saves this as
+        # "markdown_with_smarty_pants", so I just look for "markdown".
+        "markdown"
+      elsif entry_type.include?("textile")
+        # This is saved as "textile_2" on my installation of MT 5.1.
+        "textile"
+      elsif entry_type == "0" || entry_type.include?("richtext")
+        # Richtext looks to me like it's saved as HTML, so I include it here.
+        "html"
+      else
+        # Other values might need custom work.
+        entry_type
+      end
+    end
+  end
+end