jekyll-reloaded 0.12

data/lib/jekyll/migrators/joomla.rb

@@ -0,0 +1,53 @@
+require 'rubygems'
+require 'sequel'
+require 'fileutils'
+require 'yaml'
+
+# NOTE: This migrator is made for Joomla 1.5 databases.
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module Jekyll
+  module Joomla
+    def self.process(dbname, user, pass, host = 'localhost', table_prefix = 'jos_', section = '1')
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+
+      FileUtils.mkdir_p("_posts")
+
+      # Reads a MySQL database via Sequel and creates a post file for each
+      # post in wp_posts that has post_status = 'publish'. This restriction is
+      # made because 'draft' posts are not guaranteed to have valid dates.
+      query = "SELECT `title`, `alias`, CONCAT(`introtext`,`fulltext`) as content, `created`, `id` FROM #{table_prefix}content WHERE state = '0' OR state = '1' AND sectionid = '#{section}'"
+
+      db[query].each do |post|
+        # Get required fields and construct Jekyll compatible name.
+        title = post[:title]
+        slug = post[:alias]
+        date = post[:created]
+        content = post[:content]
+        name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day,
+                                               slug]
+
+        # Get the relevant fields as a hash, delete empty fields and convert
+        # to YAML for the header.
+        data = {
+           'layout' => 'post',
+           'title' => title.to_s,
+           'joomla_id' => post[:id],
+           'joomla_url' => post[:alias],
+           'date' => date
+         }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
+
+        # Write out the data and content to file
+        File.open("_posts/#{name}", "w") do |f|
+          f.puts data
+          f.puts "---"
+          f.puts content
+        end
+      end
+    end
+  end
+end

data/lib/jekyll/migrators/marley.rb

@@ -0,0 +1,52 @@
+require 'yaml'
+require 'fileutils'
+
+module Jekyll
+  module Marley
+    def self.regexp
+      { :id    => /^\d{0,4}-{0,1}(.*)$/,
+        :title => /^#\s*(.*)\s+$/,
+        :title_with_date => /^#\s*(.*)\s+\(([0-9\/]+)\)$/,
+        :published_on => /.*\s+\(([0-9\/]+)\)$/,
+        :perex => /^([^\#\n]+\n)$/,
+        :meta  => /^\{\{\n(.*)\}\}\n$/mi # Multiline Regexp 
+      }
+    end
+
+    def self.process(marley_data_dir)
+      raise ArgumentError, "marley dir #{marley_data_dir} not found" unless File.directory?(marley_data_dir)
+
+      FileUtils.mkdir_p "_posts"
+
+      posts = 0
+      Dir["#{marley_data_dir}/**/*.txt"].each do |f|
+        next unless File.exists?(f)
+
+        #copied over from marley's app/lib/post.rb
+        file_content  = File.read(f)
+        meta_content  = file_content.slice!( self.regexp[:meta] )
+        body          = file_content.sub( self.regexp[:title], '').sub( self.regexp[:perex], '').strip
+   
+        title = file_content.scan( self.regexp[:title] ).first.to_s.strip
+        prerex = file_content.scan( self.regexp[:perex] ).first.to_s.strip
+        published_on = DateTime.parse( post[:published_on] ) rescue File.mtime( File.dirname(f) )
+        meta          = ( meta_content ) ? YAML::load( meta_content.scan( self.regexp[:meta]).to_s ) : {}
+        meta['title'] = title
+        meta['layout'] = 'post'
+
+        formatted_date = published_on.strftime('%Y-%m-%d')
+        post_name =  File.dirname(f).split(%r{/}).last.gsub(/\A\d+-/, '')
+
+        name = "#{formatted_date}-#{post_name}" 
+        File.open("_posts/#{name}.markdown", "w") do |f|
+          f.puts meta.to_yaml
+          f.puts "---\n"
+          f.puts "\n#{prerex}\n\n" if prerex
+          f.puts body
+        end
+        posts += 1
+      end
+      "Created #{posts} posts!"
+    end
+  end
+end

data/lib/jekyll/migrators/mephisto.rb

@@ -0,0 +1,84 @@
+# Quickly hacked together my Michael Ivey
+# Based on mt.rb by Nick Gerakines, open source and publically
+# available under the MIT license. Use this module at your own risk.
+
+require 'rubygems'
+require 'sequel'
+require 'fastercsv'
+require 'fileutils'
+require File.join(File.dirname(__FILE__),"csv.rb")
+
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module Jekyll
+  module Mephisto
+    #Accepts a hash with database config variables, exports mephisto posts into a csv
+    #export PGPASSWORD if you must
+    def self.postgres(c)
+      sql = <<-SQL
+      BEGIN;
+      CREATE TEMP TABLE jekyll AS
+        SELECT title, permalink, body, published_at, filter FROM contents
+        WHERE user_id = 1 AND type = 'Article' ORDER BY published_at;
+      COPY jekyll TO STDOUT WITH CSV HEADER;
+      ROLLBACK;
+      SQL
+      command = %Q(psql -h #{c[:host] || "localhost"} -c "#{sql.strip}" #{c[:database]} #{c[:username]} -o #{c[:filename] || "posts.csv"})
+      puts command
+      `#{command}`
+      CSV.process
+    end
+
+    # This query will pull blog posts from all entries across all blogs. If
+    # you've got unpublished, deleted or otherwise hidden posts please sift
+    # through the created posts to make sure nothing is accidently published.
+    QUERY = "SELECT id, \
+                    permalink, \
+                    body, \
+                    published_at, \
+                    title \
+             FROM contents \
+             WHERE user_id = 1 AND \
+                   type = 'Article' AND \
+                   published_at IS NOT NULL \
+             ORDER BY published_at"
+
+    def self.process(dbname, user, pass, host = 'localhost')
+      db = Sequel.mysql(dbname, :user => user,
+                                :password => pass,
+                                :host => host,
+                                :encoding => 'utf8')
+
+      FileUtils.mkdir_p "_posts"
+
+      db[QUERY].each do |post|
+        title = post[:title]
+        slug = post[:permalink]
+        date = post[:published_at]
+        content = post[:body]
+
+        # Ideally, this script would determine the post format (markdown,
+        # html, etc) and create files with proper extensions. At this point
+        # it just assumes that markdown will be acceptable.
+        name = [date.year, date.month, date.day, slug].join('-') + ".markdown"
+
+        data = {
+           'layout' => 'post',
+           'title' => title.to_s,
+           'mt_id' => post[:entry_id],
+         }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+
+        File.open("_posts/#{name}", "w") do |f|
+          f.puts data
+          f.puts "---"
+          f.puts content
+        end
+      end
+
+    end
+  end
+end

data/lib/jekyll/migrators/mt.rb

@@ -0,0 +1,86 @@
+# Created by Nick Gerakines, open source and publically available under the
+# MIT license. Use this module at your own risk.
+# I'm an Erlang/Perl/C++ guy so please forgive my dirty ruby.
+
+require 'rubygems'
+require 'sequel'
+require 'fileutils'
+require 'yaml'
+
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module Jekyll
+  module MT
+    # This query will pull blog posts from all entries across all blogs. If
+    # you've got unpublished, deleted or otherwise hidden posts please sift
+    # through the created posts to make sure nothing is accidently published.
+    QUERY = "SELECT entry_id, \
+                    entry_basename, \
+                    entry_text, \
+                    entry_text_more, \
+                    entry_authored_on, \
+                    entry_title, \
+                    entry_convert_breaks \
+             FROM mt_entry"
+
+    def self.process(dbname, user, pass, host = 'localhost')
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+
+      FileUtils.mkdir_p "_posts"
+
+      db[QUERY].each do |post|
+        title = post[:entry_title]
+        slug = post[:entry_basename].gsub(/_/, '-')
+        date = post[:entry_authored_on]
+        content = post[:entry_text]
+        more_content = post[:entry_text_more]
+        entry_convert_breaks = post[:entry_convert_breaks]
+
+        # Be sure to include the body and extended body.
+        if more_content != nil
+          content = content + " \n" + more_content
+        end
+
+        # Ideally, this script would determine the post format (markdown,
+        # html, etc) and create files with proper extensions. At this point
+        # it just assumes that markdown will be acceptable.
+        name = [date.year, date.month, date.day, slug].join('-') + '.' +
+               self.suffix(entry_convert_breaks)
+
+        data = {
+           'layout' => 'post',
+           'title' => title.to_s,
+           'mt_id' => post[:entry_id],
+           'date' => date
+         }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
+
+        File.open("_posts/#{name}", "w") do |f|
+          f.puts data
+          f.puts "---"
+          f.puts content
+        end
+      end
+    end
+
+    def self.suffix(entry_type)
+      if entry_type.nil? || entry_type.include?("markdown")
+        # The markdown plugin I have saves this as
+        # "markdown_with_smarty_pants", so I just look for "markdown".
+        "markdown"
+      elsif entry_type.include?("textile")
+        # This is saved as "textile_2" on my installation of MT 5.1.
+        "textile"
+      elsif entry_type == "0" || entry_type.include?("richtext")
+        # Richtext looks to me like it's saved as HTML, so I include it here.
+        "html"
+      else
+        # Other values might need custom work.
+        entry_type
+      end
+    end
+  end
+end

data/lib/jekyll/migrators/posterous.rb

@@ -0,0 +1,67 @@
+require 'rubygems'
+require 'jekyll'
+require 'fileutils'
+require 'net/http'
+require 'uri'
+require "json"
+
+# ruby -r './lib/jekyll/migrators/posterous.rb' -e 'Jekyll::Posterous.process(email, pass, api_key, blog)'
+
+module Jekyll
+  module Posterous
+    def self.fetch(uri_str, limit = 10)
+      # You should choose better exception.
+      raise ArgumentError, 'Stuck in a redirect loop. Please double check your email and password' if limit == 0
+
+      response = nil
+      Net::HTTP.start('posterous.com') do |http|
+        req = Net::HTTP::Get.new(uri_str)
+        req.basic_auth @email, @pass
+        response = http.request(req)
+      end
+
+      case response
+        when Net::HTTPSuccess     then response
+        when Net::HTTPRedirection then fetch(response['location'], limit - 1)
+        else response.error!
+      end
+    end
+
+    def self.process(email, pass, api_token, blog = 'primary')
+      @email, @pass, @api_token = email, pass, api_token
+      FileUtils.mkdir_p "_posts"
+
+      posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}").body)
+      page = 1
+
+      while posts.any?
+        posts.each do |post|
+          title = post["title"]
+          slug = title.gsub(/[^[:alnum:]]+/, '-').downcase
+          date = Date.parse(post["display_date"])
+          content = post["body_html"]
+          published = !post["is_private"]
+          name = "%02d-%02d-%02d-%s.html" % [date.year, date.month, date.day, slug]
+
+          # Get the relevant fields as a hash, delete empty fields and convert
+          # to YAML for the header
+          data = {
+             'layout' => 'post',
+             'title' => title.to_s,
+             'published' => published
+           }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+
+          # Write out the data and content to file
+          File.open("_posts/#{name}", "w") do |f|
+            f.puts data
+            f.puts "---"
+            f.puts content
+          end
+        end
+
+        page += 1
+        posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}&page=#{page}").body)
+      end
+    end
+  end
+end

data/lib/jekyll/migrators/rss.rb

@@ -0,0 +1,47 @@
+# Created by Kendall Buchanan (https://github.com/kendagriff) on 2011-12-22.
+# Use at your own risk. The end.
+#
+# Usage:
+#   (URL)
+#   ruby -r '_import/rss.rb' -e "Jekyll::MigrateRSS.process('http://yourdomain.com/your-favorite-feed.xml')"
+#
+#   (Local file)
+#   ruby -r '_import/rss.rb' -e "Jekyll::MigrateRSS.process('./somefile/on/your/computer.xml')"
+
+require 'rubygems'
+require 'rss/1.0'
+require 'rss/2.0'
+require 'open-uri'
+require 'fileutils'
+require 'yaml'
+
+module Jekyll
+  module MigrateRSS
+
+    # The `source` argument may be a URL or a local file.
+    def self.process(source)
+      content = ""
+      open(source) { |s| content = s.read }
+      rss = RSS::Parser.parse(content, false)
+
+      raise "There doesn't appear to be any RSS items at the source (#{source}) provided." unless rss
+
+      rss.items.each do |item|
+        formatted_date = item.date.strftime('%Y-%m-%d')
+        post_name = item.title.split(%r{ |!|/|:|&|-|$|,}).map { |i| i.downcase if i != '' }.compact.join('-')
+        name = "#{formatted_date}-#{post_name}" 
+
+        header = {
+          'layout' => 'post',
+          'title' => item.title
+        }
+
+        File.open("_posts/#{name}.html", "w") do |f|
+          f.puts header.to_yaml
+          f.puts "---\n"
+          f.puts item.description
+        end
+      end
+    end
+  end
+end

data/lib/jekyll/migrators/textpattern.rb

@@ -0,0 +1,58 @@
+require 'rubygems'
+require 'sequel'
+require 'fileutils'
+require 'yaml'
+
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module Jekyll
+  module TextPattern
+    # Reads a MySQL database via Sequel and creates a post file for each post.
+    # The only posts selected are those with a status of 4 or 5, which means
+    # "live" and "sticky" respectively.
+    # Other statuses are 1 => draft, 2 => hidden and 3 => pending.
+    QUERY = "SELECT Title, \
+                    url_title, \
+                    Posted, \
+                    Body, \
+                    Keywords \
+             FROM textpattern \
+             WHERE Status = '4' OR \
+                   Status = '5'"
+
+    def self.process(dbname, user, pass, host = 'localhost')
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+
+      FileUtils.mkdir_p "_posts"
+
+      db[QUERY].each do |post|
+        # Get required fields and construct Jekyll compatible name.
+        title = post[:Title]
+        slug = post[:url_title]
+        date = post[:Posted]
+        content = post[:Body]
+
+        name = [date.strftime("%Y-%m-%d"), slug].join('-') + ".textile"
+
+        # Get the relevant fields as a hash, delete empty fields and convert
+        # to YAML for the header.
+        data = {
+           'layout' => 'post',
+           'title' => title.to_s,
+           'tags' => post[:Keywords].split(',')
+         }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+
+        # Write out the data and content to file.
+        File.open("_posts/#{name}", "w") do |f|
+          f.puts data
+          f.puts "---"
+          f.puts content
+        end
+      end
+    end
+  end
+end

data/lib/jekyll/migrators/tumblr.rb

@@ -0,0 +1,195 @@
+require 'rubygems'
+require 'open-uri'
+require 'fileutils'
+require 'nokogiri'
+require 'date'
+require 'json'
+require 'uri'
+require 'jekyll'
+
+module Jekyll
+  module Tumblr
+    def self.process(url, format = "html", grab_images = false,
+                     add_highlights = false, rewrite_urls = true)
+      @grab_images = grab_images
+      FileUtils.mkdir_p "_posts/tumblr"
+      url += "/api/read/json/"
+      per_page = 50
+      posts = []
+      # Two passes are required so that we can rewrite URLs.
+      # First pass builds up an array of each post as a hash.
+      begin
+        current_page = (current_page || -1) + 1
+        feed = open(url + "?num=#{per_page}&start=#{current_page * per_page}")
+        json = feed.readlines.join("\n")[21...-2]  # Strip Tumblr's JSONP chars.
+        blog = JSON.parse(json)
+        puts "Page: #{current_page + 1} - Posts: #{blog["posts"].size}"
+        posts += blog["posts"].map { |post| post_to_hash(post, format) }
+      end until blog["posts"].size < per_page
+      # Rewrite URLs and create redirects.
+      posts = rewrite_urls_and_redirects posts if rewrite_urls
+      # Second pass for writing post files.
+      posts.each do |post|
+        if format == "md"
+          post[:content] = html_to_markdown post[:content]
+          post[:content] = add_syntax_highlights post[:content] if add_highlights
+        end
+        File.open("_posts/tumblr/#{post[:name]}", "w") do |f|
+          f.puts post[:header].to_yaml + "---\n" + post[:content]
+        end
+      end
+    end
+
+    private
+
+    # Converts each type of Tumblr post to a hash with all required
+    # data for Jekyll.
+    def self.post_to_hash(post, format)
+      case post['type']
+        when "regular"
+          title = post["regular-title"]
+          content = post["regular-body"]
+        when "link"
+          title = post["link-text"] || post["link-url"]
+          content = "<a href=\"#{post["link-url"]}\">#{title}</a>"
+          unless post["link-description"].nil?
+            content << "<br/>" + post["link-description"]
+          end
+        when "photo"
+          title = post["photo-caption"]
+          max_size = post.keys.map{ |k| k.gsub("photo-url-", "").to_i }.max
+          url = post["photo-url"] || post["photo-url-#{max_size}"]
+          ext = "." + post[post.keys.select { |k|
+            k =~ /^photo-url-/ && post[k].split("/").last =~ /\./
+          }.first].split(".").last
+          content = "<img src=\"#{save_file(url, ext)}\"/>"
+          unless post["photo-link-url"].nil?
+            content = "<a href=\"#{post["photo-link-url"]}\">#{content}</a>"
+          end
+        when "audio"
+          if !post["id3-title"].nil?
+            title = post["id3-title"]
+            content = post.at["audio-player"] + "<br/>" + post["audio-caption"]
+          else
+            title = post["audio-caption"]
+            content = post.at["audio-player"]
+          end
+        when "quote"
+          title = post["quote-text"]
+          content = "<blockquote>#{post["quote-text"]}</blockquote>"
+          unless post["quote-source"].nil?
+            content << "&#8212;" + post["quote-source"]
+          end
+        when "conversation"
+          title = post["conversation-title"]
+          content = "<section><dialog>"
+          post["conversation"]["line"].each do |line|
+            content << "<dt>#{line['label']}</dt><dd>#{line}</dd>"
+          end
+          content << "</section></dialog>"
+        when "video"
+          title = post["video-title"]
+          content = post["video-player"]
+          unless post["video-caption"].nil?
+            content << "<br/>" + post["video-caption"]
+          end
+      end
+      date = Date.parse(post['date']).to_s
+      title = Nokogiri::HTML(title).text
+      slug = title.downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '')
+      {
+        :name => "#{date}-#{slug}.#{format}",
+        :header => {
+          "layout" => "post",
+          "title" => title,
+          "tags" => post["tags"],
+        },
+        :content => content,
+        :url => post["url"],
+        :slug => post["url-with-slug"],
+      }
+    end
+
+    # Create a Hash of old urls => new urls, for rewriting and
+    # redirects, and replace urls in each post. Instantiate Jekyll
+    # site/posts to get the correct permalink format.
+    def self.rewrite_urls_and_redirects(posts)
+      site = Jekyll::Site.new(Jekyll.configuration({}))
+      dir = File.join(File.dirname(__FILE__), "..")
+      urls = Hash[posts.map { |post|
+        # Create an initial empty file for the post so that
+        # we can instantiate a post object.
+        File.open("_posts/tumblr/#{post[:name]}", "w")
+        tumblr_url = URI.parse(post[:slug]).path
+        jekyll_url = Jekyll::Post.new(site, dir, "", "tumblr/" + post[:name]).url
+        redirect_dir = tumblr_url.sub(/\//, "") + "/"
+        FileUtils.mkdir_p redirect_dir
+        File.open(redirect_dir + "index.html", "w") do |f|
+          f.puts "<html><head><meta http-equiv='Refresh' content='0; " +
+                 "url=#{jekyll_url}'></head><body></body></html>"
+        end
+        [tumblr_url, jekyll_url]
+      }]
+      posts.map { |post|
+        urls.each do |tumblr_url, jekyll_url|
+          post[:content].gsub!(/#{tumblr_url}/i, jekyll_url)
+        end
+        post
+      }
+    end
+
+    # Uses Python's html2text to convert a post's content to
+    # markdown. Preserve HTML tables as per the markdown docs.
+    def self.html_to_markdown(content)
+      preserve = ["table", "tr", "th", "td"]
+      preserve.each do |tag|
+        content.gsub!(/<#{tag}/i, "$$" + tag)
+        content.gsub!(/<\/#{tag}/i, "||" + tag)
+      end
+      content = %x[echo '#{content.gsub("'", "''")}' | html2text]
+      preserve.each do |tag|
+        content.gsub!("$$" + tag, "<" + tag)
+        content.gsub!("||" + tag, "</" + tag)
+      end
+      content
+    end
+
+    # Adds pygments highlight tags to code blocks in posts that use
+    # markdown format. This doesn't guess the language of the code
+    # block, so you should modify this to suit your own content.
+    # For example, my code block only contain Python and JavaScript,
+    # so I can assume the block is JavaScript if it contains a
+    # semi-colon.
+    def self.add_syntax_highlights(content)
+      lines = content.split("\n")
+      block, indent, lang, start = false, /^    /, nil, nil
+      lines.each_with_index do |line, i|
+        if !block && line =~ indent
+          block = true
+          lang = "python"
+          start = i
+        elsif block
+          lang = "javascript" if line =~ /;$/
+          block = line =~ indent && i < lines.size - 1 # Also handle EOF
+          if !block
+            lines[start] = "{% highlight #{lang} %}"
+            lines[i - 1] = "{% endhighlight %}"
+          end
+          lines[i] = lines[i].sub(indent, "")
+        end
+      end
+      lines.join("\n")
+    end
+
+    def self.save_file(url, ext)
+      if @grab_images
+        path = "tumblr_files/#{url.split('/').last}"
+        path += ext unless path =~ /#{ext}$/
+        FileUtils.mkdir_p "tumblr_files"
+        File.open(path, "w") { |f| f.write(open(url).read) }
+        url = "/" + path
+      end
+      url
+    end
+  end
+end