jekyll-import 0.1.0.beta3 → 0.1.0.beta4

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YjI5ZmY4ODUyNjIxNTdlNmE1YjllMzVlOWJmYTc0Yjg3OGYyMWUwMA==
-  data.tar.gz: !binary |-
-    NTg5OWNkNTJmYjVmYzczMzFlYzQ5NDE3MmJlZTgxNmI4YjAwMTAzOQ==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    NjhlNzU5ZTVlN2Y1NGJiYmU4N2IxMWMyOWU2MzliMDkyMjdhMWM1NzZjZGZh
-    MTViMGE2NDU4MDQyODQ4MWI3M2FhNjNlZGQ2OGQ3MTg0ODc3OGI4NDBkM2U1
-    ZmI0Y2E1ZjM3ZjFlMGVjYzZlNTI4YThlOTRhZWFkZGQwMzk0NDc=
-  data.tar.gz: !binary |-
-    YjVkY2UyYmY3NzFkOGMwYjY0MzE1NzBiNTk5ODgwYTFmMDg3ZDI3MjExNDM3
-    ZWMyMGMzZmRmN2IwNTcxMmYwMGIzYWMxMjcyNTJlM2QwZjk2ZTg2YmMzYTgw
-    YTNiOTM1NjdmZGZlM2Q4ZDRjYTcxMzlhYjViOGRmMTNlMzZiZGE=
+SHA1:
+  metadata.gz: cbe997aa363075c0bba1d699dd1b4ccbcd29f080
+  data.tar.gz: 6856676d49ecf6a8c64eebd4f85bd846696e63f4
+SHA512:
+  metadata.gz: 303be81dcf7b40d878d25153ced30e8865602c8fe0b6b8f57111c021accd27ce3b6e5df2cb95290a2f23fa66fbdcdf791fa79cbdc527ba2851435a4d142902cf
+  data.tar.gz: 5812ffbedffcf64ce640c8af5d1140347d71406e2020e75fece71797b4f744b7e8ab29702a29d52ca27dbd11eda12a58698b36015dde65e287b869c41eb859ee

data/History.markdown

@@ -6,8 +6,11 @@
   * Improve MovableType importer (#13)
   * Add an importer for Google Reader blog exports (#36)
   * Remove dependency on html2text in the tumblr importer (#33)
+  * Add the ability to import .jrnl files (#51)
+  * Handle missing gems a bit more gracefully (#59)
 
 ### Minor Enhancements
+  * Add date and redirection pages for blogs imported from Tumblr (#54)
   * Various Tumblr Enhancements (#27)
   * Adding tags to Typo and forcing their encoding to UTF-8 (#11)
   * S9Y Importer: specify data source using --source option (#18)
@@ -16,6 +19,9 @@
     importer (#31)
   * Use tumblr slug for post is available, use that instead (#39, #40)
   * Drupal 7 importer should use latest revision of a post (#38)
+  * Improve the handling of tags in the Drupal 6 importer. Tags with
+    spaces are handled now and the importer doesn't eat tags anymore. (#42)
+  * Upgrade to `jekyll ~> 1.3` and `safe_yaml ~> 0.9.7` 
 
 ### Bug Fixes
   * Remove usage of `Hash#at` in Tumblr importer (#14)
@@ -27,8 +33,20 @@
   * Fix Drupal 7 importer so it compares node ID's properly between `node` and
     `field_data_body` tables (#38)
   * Fix prefix replacement for Drupal6 (#41)
+  * Fix an exception when a Movable Type blog did not have additional
+    entry text (#45)
+  * Create `_layouts/` before writing refresh.html in Drupal migrators (#48)
+  * Fix bug where post date in `MT` importer was not imported for older versions
+    of MT sites (#62)
+  * Fix interface of importers' `#process` method (#69)
 
 ### Site Enhancements
 
 ### Development Fixes
   * Update usage docs in RSS importer (#35)
+  * Added initial version of a test case for Tumblr (#43)
+  * Remove some outdated comments in the Drupal migrators (#50)
+  * Update the README to be more informative (#52)
+  * Add comment to Wordpress importer on how to install mysql with
+    MacPorts (#56)
+  * Correcting the homepage URL so links from rubygems.org will work (#63)

data/README.markdown

@@ -1,7 +1,18 @@
 jekyll-import
 =============
 
-The Jekyll import command for importing from various blogs to Jekyll format.
+The new __Jekyll__ command for importing from various blogs to Jekyll format.
+
+**Note: _migrators_ are now called _importers_ and are only available if one installs the `jekyll-import` _gem_.**
+
+How `jekyll-import` works:
+---
+
+Install the _rubygem_ with `gem install jekyll-import`
+* In your IRB/Ruby configurations, `require jekyll-import`
+* For CLI uses, the newly installed CLI command is now a Jekyll plugin/sub-command invoked with `jekyll import`
+
+Read [Blog migrations](http://jekyllrb.com/docs/migrations/) for more detailed info.
 
 [![Build
 Status](https://travis-ci.org/jekyll/jekyll-import.png?branch=master)](https://travis-ci.org/jekyll/jekyll-import)

data/jekyll-import.gemspec

@@ -5,8 +5,8 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 1.9.2'
 
   s.name              = 'jekyll-import'
-  s.version           = '0.1.0.beta3'
-  s.date              = '2013-07-14'
+  s.version           = '0.1.0.beta4'
+  s.date              = '2013-11-09'
   s.rubyforge_project = 'jekyll-import'
 
   s.summary     = "Import command for Jekyll (static site generator)."
@@ -14,23 +14,23 @@ Gem::Specification.new do |s|
 
   s.authors  = ["Tom Preston-Werner"]
   s.email    = 'tom@mojombo.com'
-  s.homepage = 'http://github.com/mojombo/jekyll-import'
+  s.homepage = 'http://github.com/jekyll/jekyll-import'
 
   s.require_paths = %w[lib]
 
   s.rdoc_options = ["--charset=UTF-8"]
   s.extra_rdoc_files = %w[README.markdown LICENSE]
 
-  s.add_runtime_dependency('jekyll', '~> 1.0')
+  s.add_runtime_dependency('jekyll', '~> 1.3')
   s.add_runtime_dependency('fastercsv')
   s.add_runtime_dependency('nokogiri')
-  s.add_runtime_dependency('safe_yaml', '~> 0.7.0')
-  
+  s.add_runtime_dependency('safe_yaml', '~> 0.9.7')
+
   # development dependencies
-  s.add_development_dependency('rake', "~> 10.0.3")
+  s.add_development_dependency('rake', "~> 10.1.0")
   s.add_development_dependency('rdoc', "~> 4.0.0")
   s.add_development_dependency('activesupport', '~> 3.2')
-  
+
   # test dependencies:
   s.add_development_dependency('redgreen', "~> 1.2")
   s.add_development_dependency('shoulda', "~> 3.3.2")
@@ -44,6 +44,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency('hpricot', "~> 0.8")
   s.add_development_dependency('mysql', "~> 2.8")
   s.add_development_dependency('pg', "~> 0.12")
+  s.add_development_dependency('mysql2', "~> 0.3")
 
   # = MANIFEST =
   s.files = %w[
@@ -54,26 +55,31 @@ Gem::Specification.new do |s|
     Rakefile
     jekyll-import.gemspec
     lib/jekyll-import.rb
+    lib/jekyll-import/importer.rb
+    lib/jekyll-import/importers.rb
+    lib/jekyll-import/importers/csv.rb
+    lib/jekyll-import/importers/drupal6.rb
+    lib/jekyll-import/importers/drupal7.rb
+    lib/jekyll-import/importers/enki.rb
+    lib/jekyll-import/importers/google_reader.rb
+    lib/jekyll-import/importers/joomla.rb
+    lib/jekyll-import/importers/jrnl.rb
+    lib/jekyll-import/importers/marley.rb
+    lib/jekyll-import/importers/mephisto.rb
+    lib/jekyll-import/importers/mt.rb
+    lib/jekyll-import/importers/posterous.rb
+    lib/jekyll-import/importers/rss.rb
+    lib/jekyll-import/importers/s9y.rb
+    lib/jekyll-import/importers/textpattern.rb
+    lib/jekyll-import/importers/tumblr.rb
+    lib/jekyll-import/importers/typo.rb
+    lib/jekyll-import/importers/wordpress.rb
+    lib/jekyll-import/importers/wordpressdotcom.rb
     lib/jekyll/commands/import.rb
-    lib/jekyll/jekyll-import/csv.rb
-    lib/jekyll/jekyll-import/drupal6.rb
-    lib/jekyll/jekyll-import/drupal7.rb
-    lib/jekyll/jekyll-import/enki.rb
-    lib/jekyll/jekyll-import/google_reader.rb
-    lib/jekyll/jekyll-import/joomla.rb
-    lib/jekyll/jekyll-import/marley.rb
-    lib/jekyll/jekyll-import/mephisto.rb
-    lib/jekyll/jekyll-import/mt.rb
-    lib/jekyll/jekyll-import/posterous.rb
-    lib/jekyll/jekyll-import/rss.rb
-    lib/jekyll/jekyll-import/s9y.rb
-    lib/jekyll/jekyll-import/textpattern.rb
-    lib/jekyll/jekyll-import/tumblr.rb
-    lib/jekyll/jekyll-import/typo.rb
-    lib/jekyll/jekyll-import/wordpress.rb
-    lib/jekyll/jekyll-import/wordpressdotcom.rb
     test/helper.rb
+    test/test_jrnl_importer.rb
     test/test_mt_importer.rb
+    test/test_tumblr_importer.rb
     test/test_wordpress_importer.rb
     test/test_wordpressdotcom_importer.rb
   ]

data/lib/jekyll-import.rb

@@ -1,7 +1,56 @@
 $:.unshift File.dirname(__FILE__) # For use/testing when no gem is installed
 require 'rubygems'
 require 'jekyll/commands/import'
+require 'jekyll/stevenson'
+require 'colorator'
+
+require 'jekyll-import/importer'
+require 'jekyll-import/importers'
 
 module JekyllImport
-  VERSION = '0.1.0.beta3'
+  VERSION = '0.1.0.beta4'
+
+  def self.logger
+    @logger ||= Jekyll::Stevenson.new
+  end
+
+  # Public: Add the subcommands for each importer
+  #
+  # cmd - the instance of Mercenary::Command from the
+  #
+  # Returns a list of valid subcommands
+  def self.add_importer_commands(cmd)
+    commands = []
+    JekyllImport::Importer.subclasses.each do |importer|
+      name = importer.to_s.split("::").last.downcase
+      commands << name
+      cmd.command(name.to_sym) do |c|
+        c.syntax "jekyll import #{name} [options]"
+        importer.specify_options(c)
+        c.action do |args, options|
+          importer.require_deps
+          importer.validate(options) if importer.respond_to?(:validate)
+          importer.process(options)
+        end
+      end
+    end
+    commands
+  end
+
+  def self.require_with_fallback(gems)
+    Array[gems].flatten.each do |gem|
+      begin
+        require gem
+      rescue LoadError
+        logger.error "Whoops! Looks like you need to install '#{gem}' before you can use this importer."
+        logger.error ""
+        logger.error "If you're using bundler:"
+        logger.error "  1. Add 'gem \"#{gem}\"' to your Gemfile"
+        logger.error "  2. Run 'bundle install'"
+        logger.error ""
+        logger.error "If you're not using bundler:"
+        logger.abort_with "  1. Run 'gem install #{gem}'."
+      end
+    end
+  end
 end

data/lib/jekyll-import/importer.rb

@@ -0,0 +1,11 @@
+module JekyllImport
+  class Importer
+    def self.inherited(base)
+      subclasses << base
+    end
+
+    def self.subclasses
+      @subclasses ||= []
+    end
+  end
+end

data/lib/jekyll-import/importers.rb

@@ -0,0 +1,10 @@
+module JekyllImport
+  module Importers
+    Dir.chdir(File.expand_path(File.join("importers"), File.dirname(__FILE__))) do
+      Dir.entries(".").each do |f|
+        next if f[0..0].eql?(".")
+        require "jekyll-import/importers/#{f}"
+      end
+    end
+  end
+end

data/lib/jekyll-import/importers/csv.rb

@@ -0,0 +1,50 @@
+module JekyllImport
+  module Importers
+    class CSV < Importer
+      def self.require_deps
+        JekyllImport.require_with_fallback(%w[
+          csv
+          fileutils
+        ])
+      end
+
+      def self.specify_options(c)
+        c.option 'file', '--file NAME', 'The CSV file to import (default: "posts.csv")'
+      end
+
+      # Reads a csv with title, permalink, body, published_at, and filter.
+      # It creates a post file for each row in the csv
+      def self.process(options)
+        file = options.fetch('file', "posts.csv")
+
+        FileUtils.mkdir_p "_posts"
+        posts = 0
+        abort "Cannot find the file '#{file}'. Aborting." unless File.file?(file)
+
+        ::CSV.foreach(file) do |row|
+          next if row[0] == "title"
+          posts += 1
+          name = build_name(row)
+          write_post(name, row[0], row[2])
+        end
+        "Created #{posts} posts!"
+      end
+
+      def self.write_post(name, title, content)
+        File.open("_posts/#{name}", "w") do |f|
+          f.puts <<-HEADER
+---
+layout: post
+title: #{title}
+---
+HEADER
+          f.puts content
+        end
+      end
+
+      def self.build_name(row)
+        row[3].split(" ")[0]+"-"+row[1]+(row[4] =~ /markdown/ ? ".markdown" : ".textile")
+      end
+    end
+  end
+end

data/lib/jekyll-import/importers/drupal6.rb

@@ -0,0 +1,139 @@
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module JekyllImport
+  module Importers
+    class Drupal6 < Importer
+      # Reads a MySQL database via Sequel and creates a post file for each story
+      # and blog node.
+      QUERY = "SELECT n.nid, \
+                      n.title, \
+                      nr.body, \
+                      n.created, \
+                      n.status, \
+                      GROUP_CONCAT( td.name SEPARATOR '|' ) AS 'tags' \
+                 FROM node_revisions AS nr, \
+                      node AS n \
+                 LEFT OUTER JOIN term_node AS tn ON tn.nid = n.nid \
+                 LEFT OUTER JOIN term_data AS td ON tn.tid = td.tid \
+                WHERE (n.type = 'blog' OR n.type = 'story' OR n.type = 'article') \
+                  AND n.vid = nr.vid \
+             GROUP BY n.nid"
+
+      def self.validate(options)
+        %w[dbname user].each do |option|
+          if options[option].nil?
+            abort "Missing mandatory option --#{option}."
+          end
+        end
+      end
+
+      def self.specify_options(c)
+        c.option 'dbname', '--dbname DB', 'Database name'
+        c.option 'user', '--user USER', 'Database user name'
+        c.option 'password', '--password PW', "Database user's password (default: '')"
+        c.option 'host', '--host HOST', 'Database host name (default: "localhost")'
+        c.option 'prefix', '--prefix PREFIX', 'Table prefix name'
+      end
+
+      def self.require_deps
+        JekyllImport.require_with_fallback(%w[
+          rubygems
+          sequel
+          fileutils
+          safe_yaml
+        ])
+      end
+
+      def self.process(options)
+        dbname = options.fetch('dbname')
+        user   = options.fetch('user')
+        pass   = options.fetch('password', "")
+        host   = options.fetch('host', "localhost")
+        prefix = options.fetch('prefix', "")
+
+        db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+
+        if prefix != ''
+          QUERY[" node "] = " " + prefix + "node "
+          QUERY[" node_revisions "] = " " + prefix + "node_revisions "
+          QUERY[" term_node "] = " " + prefix + "term_node "
+          QUERY[" term_data "] = " " + prefix + "term_data "
+        end
+
+        FileUtils.mkdir_p "_posts"
+        FileUtils.mkdir_p "_drafts"
+        FileUtils.mkdir_p "_layouts"
+
+        # Create the refresh layout
+        # Change the refresh url if you customized your permalink config
+        File.open("_layouts/refresh.html", "w") do |f|
+          f.puts <<EOF
+<!DOCTYPE html>
+<html>
+<head>
+<meta http-equiv="content-type" content="text/html; charset=utf-8" />
+<meta http-equiv="refresh" content="0;url={{ page.refresh_to_post_id }}.html" />
+</head>
+</html>
+EOF
+        end
+
+        db[QUERY].each do |post|
+          # Get required fields and construct Jekyll compatible name
+          node_id = post[:nid]
+          title = post[:title]
+          content = post[:body]
+          tags = post.fetch(:tags, '').downcase.strip
+          created = post[:created]
+          time = Time.at(created)
+          is_published = post[:status] == 1
+          dir = is_published ? "_posts" : "_drafts"
+          slug = title.strip.downcase.gsub(/(&|&amp;)/, ' and ').gsub(/[\s\.\/\\]/, '-').gsub(/[^\w-]/, '').gsub(/[-_]{2,}/, '-').gsub(/^[-_]/, '').gsub(/[-_]$/, '')
+          name = time.strftime("%Y-%m-%d-") + slug + '.md'
+
+          # Get the relevant fields as a hash, delete empty fields and convert
+          # to YAML for the header
+          data = {
+             'layout' => 'post',
+             'title' => title.to_s,
+             'created' => created,
+             'categories' => tags.split('|')
+           }.delete_if { |k,v| v.nil? || v == ''}.each_pair {
+              |k,v| ((v.is_a? String) ? v.force_encoding("UTF-8") : v)
+           }.to_yaml
+
+          # Write out the data and content to file
+          File.open("#{dir}/#{name}", "w") do |f|
+            f.puts data
+            f.puts "---"
+            f.puts content
+          end
+
+          # Make a file to redirect from the old Drupal URL
+          if is_published
+            aliases = db["SELECT dst FROM #{prefix}url_alias WHERE src = ?", "node/#{node_id}"].all
+
+            aliases.push(:dst => "node/#{node_id}")
+
+            aliases.each do |url_alias|
+              FileUtils.mkdir_p url_alias[:dst]
+              File.open("#{url_alias[:dst]}/index.md", "w") do |f|
+                f.puts "---"
+                f.puts "layout: refresh"
+                f.puts "refresh_to_post_id: /#{time.strftime("%Y/%m/%d/") + slug}"
+                f.puts "---"
+              end
+            end
+          end
+        end
+
+        # TODO: Make dirs & files for nodes of type 'page'
+        # Make refresh pages for these as well
+      end
+    end
+  end
+end