jekyll-import 0.1.0.beta1

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/History.txt

@@ -0,0 +1,7 @@
+== HEAD
+  * Major Enhancements
+    * Improve MovableType importer (#13)
+  * Minor Enhancements
+  * Bug Fixes
+  * Site Enhancements
+  * Development fixes

data/LICENSE

@@ -0,0 +1,21 @@
+(The MIT License)
+
+Copyright (c) 2013 Tom Preston-Werner
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the 'Software'), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,4 @@
+jekyll-import
+=============
+
+The Jekyll import command for importing from various blogs to Jekyll format.

data/Rakefile

@@ -0,0 +1,151 @@
+# require 'rake/testtask'
+
+# Rake::TestTask.new do |t|
+#   t.libs << 'test'
+# end
+
+# desc "Run tests"
+# task :default => :test
+
+require 'rubygems'
+require 'rake'
+require 'date'
+
+#############################################################################
+#
+# Helper functions
+#
+#############################################################################
+
+def name
+  @name ||= Dir['*.gemspec'].first.split('.').first
+end
+
+def version
+  line = File.read("lib/#{name}.rb")[/^\s*VERSION\s*=\s*.*/]
+  line.match(/.*VERSION\s*=\s*['"](.*)['"]/)[1]
+end
+
+def date
+  Date.today.to_s
+end
+
+def rubyforge_project
+  name
+end
+
+def gemspec_file
+  "#{name}.gemspec"
+end
+
+def gem_file
+  "#{name}-#{version}.gem"
+end
+
+def replace_header(head, header_name)
+  head.sub!(/(\.#{header_name}\s*= ').*'/) { "#{$1}#{send(header_name)}'"}
+end
+
+#############################################################################
+#
+# Standard tasks
+#
+#############################################################################
+
+task :default => :test
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "#{name} #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+desc "Open an irb session preloaded with this library"
+task :console do
+  sh "irb -rubygems -r ./lib/#{name}.rb"
+end
+
+#############################################################################
+#
+# Custom tasks (add your own tasks here)
+#
+#############################################################################
+
+namespace :migrate do
+  desc "Migrate from mephisto in the current directory"
+  task :mephisto do
+    sh %q(ruby -r './lib/jekyll/migrators/mephisto' -e 'Jekyll::Mephisto.postgres(:database => "#{ENV["SERVER"]}", "#{ENV["DB"]}")')
+  end
+  desc "Migrate from Movable Type in the current directory"
+  task :mt do
+    sh %q(ruby -r './lib/jekyll/migrators/mt' -e 'Jekyll::MT.process("#{ENV["DB"]}", "#{ENV["USER"]}", "#{ENV["PASS"]}")')
+  end
+  desc "Migrate from Typo in the current directory"
+  task :typo do
+    sh %q(ruby -r './lib/jekyll/migrators/typo' -e 'Jekyll::Typo.process("#{ENV["DB"]}", "#{ENV["USER"]}", "#{ENV["PASS"]}")')
+  end
+end
+
+#############################################################################
+#
+# Packaging tasks
+#
+#############################################################################
+
+desc "Create tag v#{version} and build and push #{gem_file} to Rubygems"
+task :release => :build do
+  unless `git branch` =~ /^\* master$/
+    puts "You must be on the master branch to release!"
+    exit!
+  end
+  sh "git commit --allow-empty -a -m 'Release #{version}'"
+  sh "git tag v#{version}"
+  sh "git push origin master"
+  sh "git push origin v#{version}"
+  sh "gem push pkg/#{name}-#{version}.gem"
+end
+
+desc "Build #{gem_file} into the pkg directory"
+task :build => :gemspec do
+  sh "mkdir -p pkg"
+  sh "gem build #{gemspec_file}"
+  sh "mv #{gem_file} pkg"
+end
+
+desc "Generate #{gemspec_file}"
+task :gemspec do
+  # read spec file and split out manifest section
+  spec = File.read(gemspec_file)
+  head, manifest, tail = spec.split("  # = MANIFEST =\n")
+
+  # replace name version and date
+  replace_header(head, :name)
+  replace_header(head, :version)
+  replace_header(head, :date)
+  #comment this out if your rubyforge_project has a different name
+  replace_header(head, :rubyforge_project)
+
+  # determine file list from git ls-files
+  files = `git ls-files`.
+    split("\n").
+    sort.
+    reject { |file| file =~ /^\./ }.
+    reject { |file| file =~ /^(rdoc|pkg)/ }.
+    map { |file| "    #{file}" }.
+    join("\n")
+
+  # piece file back together and write
+  manifest = "  s.files = %w[\n#{files}\n  ]\n"
+  spec = [head, manifest, tail].join("  # = MANIFEST =\n")
+  File.open(gemspec_file, 'w') { |io| io.write(spec) }
+  puts "Updated #{gemspec_file}"
+end

data/jekyll-import.gemspec

@@ -0,0 +1,80 @@
+Gem::Specification.new do |s|
+  s.specification_version = 2 if s.respond_to? :specification_version=
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.rubygems_version = '1.3.5'
+
+  s.name              = 'jekyll-import'
+  s.version           = '0.1.0.beta1'
+  s.date              = '2013-03-19'
+  s.rubyforge_project = 'jekyll-import'
+
+  s.summary     = "Import command for Jekyll (static site generator)."
+  s.description = "Provides the Import command for Jekyll."
+
+  s.authors  = ["Tom Preston-Werner"]
+  s.email    = 'tom@mojombo.com'
+  s.homepage = 'http://github.com/mojombo/jekyll-import'
+
+  s.require_paths = %w[lib]
+
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.extra_rdoc_files = %w[README.md LICENSE]
+
+  s.add_runtime_dependency('jekyll')
+  s.add_runtime_dependency('fastercsv')
+  s.add_runtime_dependency('nokogiri')
+  s.add_runtime_dependency('safe_yaml', '~> 0.7')
+  
+  # development dependencies
+  s.add_development_dependency('rake', "~> 10.0.3")
+  s.add_development_dependency('rdoc', "~> 4.0.0")
+  
+  # test dependencies:
+  s.add_development_dependency('redgreen', "~> 1.2")
+  s.add_development_dependency('shoulda', "~> 3.3.2")
+  s.add_development_dependency('rr', "~> 1.0")
+  s.add_development_dependency('simplecov', "~> 0.7")
+  s.add_development_dependency('simplecov-gem-adapter', "~> 1.0.1")
+
+  # migrator dependencies:
+  s.add_development_dependency('sequel', "~> 3.42")
+  s.add_development_dependency('htmlentities', "~> 4.3")
+  s.add_development_dependency('hpricot', "~> 0.8")
+  s.add_development_dependency('mysql', "~> 2.8")
+  s.add_development_dependency('pg', "~> 0.12")
+
+  # = MANIFEST =
+  s.files = %w[
+    Gemfile
+    History.txt
+    LICENSE
+    README.md
+    Rakefile
+    jekyll-import.gemspec
+    lib/jekyll-import.rb
+    lib/jekyll/commands/import.rb
+    lib/jekyll/jekyll-import/csv.rb
+    lib/jekyll/jekyll-import/drupal6.rb
+    lib/jekyll/jekyll-import/drupal7.rb
+    lib/jekyll/jekyll-import/enki.rb
+    lib/jekyll/jekyll-import/joomla.rb
+    lib/jekyll/jekyll-import/marley.rb
+    lib/jekyll/jekyll-import/mephisto.rb
+    lib/jekyll/jekyll-import/mt.rb
+    lib/jekyll/jekyll-import/posterous.rb
+    lib/jekyll/jekyll-import/rss.rb
+    lib/jekyll/jekyll-import/s9y.rb
+    lib/jekyll/jekyll-import/textpattern.rb
+    lib/jekyll/jekyll-import/tumblr.rb
+    lib/jekyll/jekyll-import/typo.rb
+    lib/jekyll/jekyll-import/wordpress.rb
+    lib/jekyll/jekyll-import/wordpressdotcom.rb
+    test/helper.rb
+    test/test_mt_importer.rb
+    test/test_wordpress_importer.rb
+    test/test_wordpressdotcom_importer.rb
+  ]
+  # = MANIFEST =
+
+  s.test_files = s.files.select { |path| path =~ /^test\/test_.*\.rb/ }
+end

data/lib/jekyll-import.rb

@@ -0,0 +1,7 @@
+$:.unshift File.dirname(__FILE__) # For use/testing when no gem is installed
+require 'rubygems'
+require 'jekyll/commands/import'
+
+module JekyllImport
+  VERSION = '0.1.0.beta1'
+end

data/lib/jekyll/commands/import.rb

@@ -0,0 +1,51 @@
+$:.unshift File.expand_path("../../", File.dirname(__FILE__)) # load from jekyll-import/lib
+require 'jekyll/command'
+
+module Jekyll
+  module Commands
+    class Import < Command
+      IMPORTERS = {
+        :csv => 'CSV',
+        :drupal6 => 'Drupal6',
+        :drupal7 => 'Drupal7',
+        :enki => 'Enki',
+        :joomla => 'Joomla',
+        :marley => 'Marley',
+        :mephisto => 'Mephisto',
+        :mt => 'MT',
+        :posterous => 'Posterous',
+        :rss => 'RSS',
+        :s9y => 'S9Y',
+        :textpattern => 'TextPattern',
+        :tumblr => 'Tumblr',
+        :typo => 'Typo',
+        :wordpress => 'WordPress',
+        :wordpressdotcom => 'WordpressDotCom'
+      }
+
+      def self.abort_on_invalid_migator
+        msg = "You must specify a valid migrator. Valid choices:\n"
+        IMPORTERS.keys.each do |k, v|
+          msg += "* #{k}\n"
+        end
+        abort msg
+      end
+
+      def self.process(migrator, options)
+        if IMPORTERS.keys.include?(migrator.to_sym)
+          migrator = migrator.downcase
+
+          require File.join(File.dirname(__FILE__), "..", "jekyll-import", "#{migrator}.rb")
+
+          if JekyllImport.const_defined?(IMPORTERS[migrator.to_sym])
+            puts 'Importing...'
+            klass = JekyllImport.const_get(IMPORTERS[migrator.to_sym])
+            klass.process(options.__hash__)
+          end
+        else
+          abort_on_invalid_migator
+        end
+      end
+    end
+  end
+end

data/lib/jekyll/jekyll-import/csv.rb

@@ -0,0 +1,26 @@
+module JekyllImport
+  module CSV
+    # Reads a csv with title, permalink, body, published_at, and filter.
+    # It creates a post file for each row in the csv
+    def self.process(file = "posts.csv")
+      FileUtils.mkdir_p "_posts"
+      posts = 0
+      FasterCSV.foreach(file) do |row|
+        next if row[0] == "title"
+        posts += 1
+        name = row[3].split(" ")[0]+"-"+row[1]+(row[4] =~ /markdown/ ? ".markdown" : ".textile")
+        File.open("_posts/#{name}", "w") do |f|
+          f.puts <<-HEADER
+---
+layout: post
+title: #{row[0]}
+---
+
+          HEADER
+          f.puts row[2]
+        end
+      end
+      "Created #{posts} posts!"
+    end
+  end
+end

data/lib/jekyll/jekyll-import/drupal6.rb

@@ -0,0 +1,102 @@
+require 'rubygems'
+require 'sequel'
+require 'fileutils'
+require 'safe_yaml'
+
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module JekyllImport
+  module Drupal6
+    # Reads a MySQL database via Sequel and creates a post file for each story
+    # and blog node in table node.
+    QUERY = "SELECT n.nid, \
+                    n.title, \
+                    nr.body, \
+                    n.created, \
+                    n.status \
+             FROM node AS n, \
+                  node_revisions AS nr \
+             WHERE (n.type = 'blog' OR n.type = 'story') \
+             AND n.vid = nr.vid"
+
+    def self.process(dbname, user, pass, host = 'localhost', prefix = '')
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+
+      if prefix != ''
+        QUERY[" node "] = " " + prefix + "node "
+        QUERY[" node_revisions "] = " " + prefix + "node_revisions "
+      end
+
+      FileUtils.mkdir_p "_posts"
+      FileUtils.mkdir_p "_drafts"
+
+      # Create the refresh layout
+      # Change the refresh url if you customized your permalink config
+      File.open("_layouts/refresh.html", "w") do |f|
+        f.puts <<EOF
+<!DOCTYPE html>
+<html>
+<head>
+<meta http-equiv="content-type" content="text/html; charset=utf-8" />
+<meta http-equiv="refresh" content="0;url={{ page.refresh_to_post_id }}.html" />
+</head>
+</html>
+EOF
+      end
+
+      db[QUERY].each do |post|
+        # Get required fields and construct Jekyll compatible name
+        node_id = post[:nid]
+        title = post[:title]
+        content = post[:body]
+        created = post[:created]
+        time = Time.at(created)
+        is_published = post[:status] == 1
+        dir = is_published ? "_posts" : "_drafts"
+        slug = title.strip.downcase.gsub(/(&|&amp;)/, ' and ').gsub(/[\s\.\/\\]/, '-').gsub(/[^\w-]/, '').gsub(/[-_]{2,}/, '-').gsub(/^[-_]/, '').gsub(/[-_]$/, '')
+        name = time.strftime("%Y-%m-%d-") + slug + '.md'
+
+        # Get the relevant fields as a hash, delete empty fields and convert
+        # to YAML for the header
+        data = {
+           'layout' => 'post',
+           'title' => title.to_s,
+           'created' => created,
+         }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+
+        # Write out the data and content to file
+        File.open("#{dir}/#{name}", "w") do |f|
+          f.puts data
+          f.puts "---"
+          f.puts content
+        end
+
+        # Make a file to redirect from the old Drupal URL
+        if is_published
+          aliases = db["SELECT dst FROM #{prefix}url_alias WHERE src = ?", "node/#{node_id}"].all
+
+          aliases.push(:dst => "node/#{node_id}")
+
+          aliases.each do |url_alias|
+            FileUtils.mkdir_p url_alias[:dst]
+            File.open("#{url_alias[:dst]}/index.md", "w") do |f|
+              f.puts "---"
+              f.puts "layout: refresh"
+              f.puts "refresh_to_post_id: /#{time.strftime("%Y/%m/%d/") + slug}"
+              f.puts "---"
+            end
+          end
+        end
+      end
+
+      # TODO: Make dirs & files for nodes of type 'page'
+        # Make refresh pages for these as well
+
+      # TODO: Make refresh dirs & files according to entries in url_alias table
+    end
+  end
+end