jekyll-import 0.1.0 → 0.2.0

data/lib/jekyll-import/importers/tumblr.rb

@@ -68,13 +68,16 @@ module JekyllImport
       # Writes a post out to disk
       def self.write_post(post, use_markdown, add_highlights)
         content = post[:content]
-        if use_markdown
-          content = html_to_markdown content
-          content = add_syntax_highlights content if add_highlights
-        end
 
-        File.open("_posts/tumblr/#{post[:name]}", "w") do |f|
-          f.puts post[:header].to_yaml + "---\n" + content
+        if content
+          if use_markdown
+            content = html_to_markdown content
+            content = add_syntax_highlights content if add_highlights
+          end
+  
+          File.open("_posts/tumblr/#{post[:name]}", "w") do |f|
+            f.puts post[:header].to_yaml + "---\n" + content
+          end
         end
       end
 
@@ -125,6 +128,9 @@ module JekyllImport
             unless post["video-caption"].nil?
               content << "<br/>" + post["video-caption"]
             end
+          when "answer"
+            title = post["question"]
+            content = post["answer"]
         end
         date = Date.parse(post['date']).to_s
         title = Nokogiri::HTML(title).text
@@ -140,7 +146,7 @@ module JekyllImport
             "layout" => "post",
             "title" => title,
             "date" => Time.parse(post['date']).xmlschema,
-            "tags" => post["tags"],
+            "tags" => (post["tags"] or []),
             "tumblr_url" => post["url-with-slug"]
           },
           :content => content,

data/lib/jekyll-import/importers/wordpress.rb

@@ -115,6 +115,7 @@ module JekyllImport
              posts.post_title    AS `title`,
              posts.post_name     AS `slug`,
              posts.post_date     AS `date`,
+             posts.post_date_gmt AS `date_gmt`,
              posts.post_content  AS `content`,
              posts.post_excerpt  AS `excerpt`,
              posts.comment_count AS `comment_count`,
@@ -276,7 +277,8 @@ module JekyllImport
           'more_anchor'   => more_anchor,
           'wordpress_id'  => post[:id],
           'wordpress_url' => post[:guid].to_s,
-          'date'          => date,
+          'date'          => date.to_s,
+          'date_gmt'      => post[:date_gmt].to_s,
           'categories'    => options[:categories] ? categories : nil,
           'tags'          => options[:tags] ? tags : nil,
           'comments'      => options[:comments] ? comments : nil,
@@ -286,7 +288,7 @@ module JekyllImport
         File.open("_posts/#{name}", "w") do |f|
           f.puts data
           f.puts "---"
-          f.puts content
+          f.puts Util.wpautop(content)
         end
       end
 

data/lib/jekyll-import/importers/wordpressdotcom.rb

@@ -26,6 +26,19 @@ module JekyllImport
         import_count = Hash.new(0)
         doc = Hpricot::XML(File.read(source))
 
+        # Fetch authors data from header
+        authors = Hash[
+          (doc/:channel/'wp:author').map do |author|
+          [author.at("wp:author_login").inner_text.strip, {
+            "login" => author.at("wp:author_login").inner_text.strip,
+            "email" => author.at("wp:author_email").inner_text,
+            "display_name" => author.at("wp:author_display_name").inner_text,
+            "first_name" => author.at("wp:author_first_name").inner_text,
+            "last_name" => author.at("wp:author_last_name").inner_text
+          }]
+          end
+        ] rescue {}
+
         (doc/:channel/:item).each do |item|
           title = item.at(:title).inner_text.strip
           permalink_title = item.at('wp:post_name').inner_text
@@ -51,9 +64,11 @@ module JekyllImport
           item.search("wp:postmeta").each do |meta|
             key = meta.at('wp:meta_key').inner_text
             value = meta.at('wp:meta_value').inner_text
-            metas[key] = value;
+            metas[key] = value
           end
 
+          author_login = item.at('dc:creator').inner_text.strip
+
           name = "#{date.strftime('%Y-%m-%d')}-#{permalink_title}.html"
           header = {
             'layout' => type,
@@ -63,7 +78,8 @@ module JekyllImport
             'status'   => status,
             'type'   => type,
             'published' => published,
-            'meta'   => metas
+            'meta'   => metas,
+            'author' => authors[author_login]
           }
 
           begin
@@ -71,7 +87,7 @@ module JekyllImport
             File.open("_#{type}s/#{name}", "w") do |f|
               f.puts header.to_yaml
               f.puts '---'
-              f.puts item.at('content:encoded').inner_text
+              f.puts Util.wpautop(item.at('content:encoded').inner_text)
             end
           rescue => e
             puts "Couldn't import post!"

data/lib/jekyll-import/util.rb

@@ -0,0 +1,76 @@
+module JekyllImport
+  module Util
+
+    # Ruby translation of wordpress wpautop (see https://core.trac.wordpress.org/browser/trunk/src/wp-includes/formatting.php)
+    #
+    # A group of regex replaces used to identify text formatted with newlines and
+    # replace double line-breaks with HTML paragraph tags. The remaining
+    # line-breaks after conversion become <<br />> tags, unless $br is set to false
+    #
+    # @param string pee The text which has to be formatted.
+    # @param bool br Optional. If set, this will convert all remaining line-breaks after paragraphing. Default true.
+    # @return string Text which has been converted into correct paragraph tags.
+    #
+    def self.wpautop(pee, br = true)
+      return '' if pee.strip == ''
+
+      allblocks = '(?:table|thead|tfoot|caption|col|colgroup|tbody|tr|td|th|div|dl|dd|dt|ul|ol|li|pre|select|option|form|map|area|blockquote|address|math|style|p|h[1-6]|hr|fieldset|noscript|legend|section|article|aside|hgroup|header|footer|nav|figure|figcaption|details|menu|summary)'
+      pre_tags = {}
+      pee = pee + "\n"
+
+      if pee.include?('<pre')
+        pee_parts = pee.split('</pre>')
+        last_pee = pee_parts.pop
+        pee = ''
+        pee_parts.each_with_index do |pee_part, i|
+          start = pee_part.index('<pre')
+
+          unless start
+            pee += pee_part
+            continue
+          end
+
+          name = "<pre wp-pre-tag-#{i}></pre>"
+          pre_tags[name] = pee_part[start..-1] + '</pre>'
+
+          pee += pee_part[0, start] + name
+        end
+        pee += last_pee
+      end
+
+      pee = pee.gsub(Regexp.new('<br />\s*<br />'), "\n\n")
+      pee = pee.gsub(Regexp.new("(<" + allblocks + "[^>]*>)"), "\n\\1")
+      pee = pee.gsub(Regexp.new("(</" + allblocks + ">)"), "\\1\n\n")
+      pee = pee.gsub("\r\n", "\n").gsub("\r", "\n")
+      if pee.include? '<object'
+        pee = pee.gsub(Regexp.new('\s*<param([^>]*)>\s*'), "<param\\1>")
+        pee = pee.gsub(Regexp.new('\s*</embed>\s*'), '</embed>')
+      end
+
+      pees = pee.split(/\n\s*\n/).compact
+      pee = ''
+      pees.each { |tinkle| pee += '<p>' + tinkle.chomp("\n") + "</p>\n" }
+      pee = pee.gsub(Regexp.new('<p>\s*</p>'), '')
+      pee = pee.gsub(Regexp.new('<p>([^<]+)</(div|address|form)>'), "<p>\\1</p></\\2>")
+      pee = pee.gsub(Regexp.new('<p>\s*(</?' + allblocks + '[^>]*>)\s*</p>'), "\\1")
+      pee = pee.gsub(Regexp.new('<p>(<li.+?)</p>'), "\\1")
+      pee = pee.gsub(Regexp.new('<p><blockquote([^>]*)>', 'i'), "<blockquote\\1><p>")
+      pee = pee.gsub('</blockquote></p>', '</p></blockquote>')
+      pee = pee.gsub(Regexp.new('<p>\s*(</?' + allblocks + '[^>]*>)'), "\\1")
+      pee = pee.gsub(Regexp.new('(</?' + allblocks + '[^>]*>)\s*</p>'), "\\1")
+      if br
+        pee = pee.gsub(Regexp.new('<(script|style).*?</\1>')) { |match| match.gsub("\n", "<WPPreserveNewline />") }
+        pee = pee.gsub(Regexp.new('(?<!<br />)\s*\n'), "<br />\n")
+        pee = pee.gsub('<WPPreserveNewline />', "\n")
+      end
+      pee = pee.gsub(Regexp.new('(</?' + allblocks + '[^>]*>)\s*<br />'), "\\1")
+      pee = pee.gsub(Regexp.new('<br />(\s*</?(?:p|li|div|dl|dd|dt|th|pre|td|ul|ol)[^>]*>)'), "\\1")
+      pee = pee.gsub(Regexp.new('\n</p>$'), '</p>')
+
+      pre_tags.each do |name, value|
+        pee.gsub!(name, value)
+      end
+      pee
+    end
+  end
+end

data/lib/jekyll-import/version.rb

@@ -0,0 +1,3 @@
+module JekyllImport
+  VERSION = '0.2.0'
+end

data/lib/jekyll/commands/import.rb

@@ -5,6 +5,7 @@ require 'jekyll-import'
 module Jekyll
   module Commands
     class Import < Command
+
       IMPORTERS = {
         :behance => 'Behance',
         :csv => 'CSV',
@@ -28,26 +29,47 @@ module Jekyll
         :wordpressdotcom => 'WordpressDotCom'
       }
 
-      def self.abort_on_invalid_migrator(migrator)
-        msg = "Sorry, '#{migrator}' isn't a valid migrator. Valid choices:\n"
-        IMPORTERS.keys.each do |k, v|
-          msg += "* #{k}\n"
+      class << self
+
+        def init_with_program(prog)
+          prog.command(:import) do |c|
+            c.syntax 'import <platform> [options]'
+            c.description 'Import your old blog to Jekyll'
+            importers = JekyllImport.add_importer_commands(c)
+
+            c.action do |args, options|
+              if args.empty?
+                Jekyll.logger.warn "You must specify an importer."
+                Jekyll.logger.info "Valid options are:"
+                importers.each { |i| Jekyll.logger.info "*", "#{i}" }
+              end
+            end
+          end
         end
-        abort msg
-      end
 
-      def self.process(migrator, options)
-        migrator = migrator.to_s.downcase
+        def process(migrator, options)
+          migrator = migrator.to_s.downcase
 
-        if IMPORTERS.keys.include?(migrator.to_sym)
-          if JekyllImport::Importers.const_defined?(IMPORTERS[migrator.to_sym])
-            klass = JekyllImport::Importers.const_get(IMPORTERS[migrator.to_sym])
-            klass.run(options.__hash__)
+          if IMPORTERS.keys.include?(migrator.to_sym)
+            if JekyllImport::Importers.const_defined?(IMPORTERS[migrator.to_sym])
+              klass = JekyllImport::Importers.const_get(IMPORTERS[migrator.to_sym])
+              klass.run(options.__hash__)
+            end
+          else
+            abort_on_invalid_migrator(migrator)
           end
-        else
-          abort_on_invalid_migrator(migrator)
         end
+
+        def abort_on_invalid_migrator(migrator)
+          msg = "Sorry, '#{migrator}' isn't a valid migrator. Valid choices:\n"
+          IMPORTERS.keys.each do |k, v|
+            msg += "* #{k}\n"
+          end
+          abort msg
+        end
+
       end
+
     end
   end
 end

data/site/_posts/2014-03-16-jekyll-import-0-2-0-released.markdown

@@ -0,0 +1,16 @@
+---
+layout: news_item
+title: 'jekyll-import 0.2.0 Released'
+date: 2014-03-16 01:05:25 -0400
+author: parkr
+version: 0.2.0
+categories: [release]
+---
+
+It seems jekyll-import has been collecting some dust lately. There are some
+great fixes and feature improvements in here so, hey, why let them lay
+unutilized? No reason at all, my dear Watson, no reason at all.
+
+Please enjoy! Check out the [GitHub Release][] for more specifics.
+
+[GitHub Release]: https://github.com/jekyll/jekyll-import/releases/tag/v0.2.0

data/site/css/style.css

@@ -220,17 +220,16 @@ body > footer a:hover img {
 .quickstart .code {
   font-size: 12px;
   display: block;
-  margin: 0 0 -30px;
+  margin: 10px 0 20px;
 }
 
 @media (min-width: 768px){
   .quickstart .code {
     font-size: 18px;
-    margin: -30px 0;
-    float: right;
+    margin: 10px 0 30px;
   }
   .quickstart h4 {
-    margin: 50px 0 0;
+    margin: 20px 0 0;
     text-align: center;
   }
 }

data/site/docs/mt.md

@@ -15,9 +15,25 @@ $ ruby -rubygems -e 'require "jekyll-import";
       "dbname"   => "name",
       "user"     => "myuser",
       "password" => "mypassword",
-      "host"     => "myhost"
+      "host"     => "myhost",
+      "comments"     => true
     })'
 {% endhighlight %}
 
+Posts will be generated and placed in `_posts` directory.
+
 The only required fields are `dbname` and `user`. `password` defaults to `""`
-and `host` defaults to `"localhost"`.
+and `host` defaults to `"localhost"`.
+
+`comments`, which defaults to false, control the generation of
+comment. If `comments` set to true, posts will be generated and placed
+in `_comments` directory.
+
+
+All of the posts and comments will include `post_id` in YAML front
+matter to link a post and its comments.
+
+To include imported comments as part of a post, use the yet to merge
+[fork of mt-static-comments](https://github.com/shigeya/jekyll-static-comments/tree/mt_static_comments)
+to include statically generate comments in your post. Fork and provide
+feedback if necessary.

data/site/docs/tumblr.md

@@ -13,12 +13,12 @@ To import your posts from [Tumblr](http://tumblr.com), run:
 $ ruby -rubygems -e 'require "jekyll-import";
     JekyllImport::Importers::Tumblr.run({
       "url"            => "http://myblog.tumblr.com",
-      "user"           => "html", # or "md"
-      "password"       => false,  # whether to download images as well.
+      "format"         => "html", # or "md"
+      "grab_images"    => false,  # whether to download images as well.
       "add_highlights" => false,  # whether to wrap code blocks (indented 4 spaces) in a Liquid "highlight" tag
       "rewrite_urls"   => false   # whether to write pages that redirect from the old Tumblr paths to the new Jekyll paths
     })'
 {% endhighlight %}
 
 The only required field is `url`. The other fields default to their above
-values.
+values.

data/site/docs/wordpress.md

@@ -29,3 +29,12 @@ $ ruby -rubygems -e 'require "jekyll-import";
 {% endhighlight %}
 
 None of the fields are required. Their defaults are as you see above.
+
+<div class="note info">
+  <h5>This only imports post &amp; page data &amp; content</h5>
+  <p>
+    This importer only converts your posts and creates YAML front-matter.
+    It does not import any layouts, styling, or external files
+    (images, CSS, etc.).
+  </p>
+</div>

data/site/index.html

@@ -37,10 +37,10 @@ overview: true
 </section>
 <section class="quickstart">
   <div class="grid">
-    <div class="unit golden-small center-on-mobiles">
+    <div class="unit whole center-on-mobiles">
       <h4>Get up and running <em>in&nbsp;seconds</em>.</h4>
     </div>
-    <div class="unit golden-large code">
+    <div class="unit whole center-on-mobiles code">
       <p class="title">Quick-start Instructions</p>
       <div class="shell">
         <p class="line">
@@ -61,12 +61,7 @@ overview: true
         <p class="line">
           <span class="path">~/my-awesome-site</span>
           <span class="prompt">$</span>
-          <span class="command">jekyll import wordpress \ </span>
-        </p>
-        <p class="line">
-          <span class="path"> </span>
-          <span class="prompt"> </span>
-          <span class="command">--source wordpress.xml</span>
+          <span class="command">jekyll import wordpressdotcom --source wordpress.xml</span>
         </p>
         <p class="line">
           <span class="path">~/my-awesome-site</span>

data/test/test_tumblr_importer.rb

@@ -3,59 +3,88 @@ require 'json'
 
 class TestTumblrImporter < Test::Unit::TestCase
 
-	context "A Tumblr blog" do
-		setup do
+  context "A Tumblr blog" do
+    setup do
       Importers::Tumblr.require_deps
-			@jsonPayload = '{"tumblelog":{"title":"JekyllImport","description":"Jekyll Importer Test.","name":"JekyllImport","timezone":"Canada\/Atlantic","cname":"https://github.com/jekyll/jekyll-import/","feeds":[]},"posts-start":0,"posts-total":"1","posts-type":false,"posts":[{"id":54759400073,"url":"https:\/\/github.com\/post\/54759400073","url-with-slug":"http:\/\/github.com\/post\/54759400073\/jekyll-test","type":"regular","date-gmt":"2013-07-06 16:27:23 GMT","date":"Sat, 06 Jul 2013 13:27:23","bookmarklet":null,"mobile":null,"feed-item":"","from-feed-id":0,"unix-timestamp":1373128043,"format":"html","reblog-key":"0L6yPcHr","slug":"jekyll-test","regular-title":"Jekyll: Test","regular-body":"<p>Testing...<\/p>","tags":["jekyll"]}]}'
-			@posts = JSON.parse(@jsonPayload)
-		end
+      @jsonPayload = '{"tumblelog":{"title":"JekyllImport","description":"Jekyll Importer Test.","name":"JekyllImport","timezone":"Canada\/Atlantic","cname":"https://github.com/jekyll/jekyll-import/","feeds":[]},"posts-start":0,"posts-total":"2","posts-type":false,"posts":[{"id":54759400073,"url":"https:\/\/github.com\/post\/54759400073","url-with-slug":"http:\/\/github.com\/post\/54759400073\/jekyll-test","type":"regular","date-gmt":"2013-07-06 16:27:23 GMT","date":"Sat, 06 Jul 2013 13:27:23","bookmarklet":null,"mobile":null,"feed-item":"","from-feed-id":0,"unix-timestamp":1373128043,"format":"html","reblog-key":"0L6yPcHr","slug":"jekyll-test","regular-title":"Jekyll: Test","regular-body":"<p>Testing...<\/p>","tags":["jekyll"]},{"id":"71845593082","url":"http:\/\/example.com\/post\/71845593082","url-with-slug":"http:\/\/example.com\/post\/71845593082\/knock-knock","type":"answer","date-gmt":"2014-01-01 14:08:45 GMT","date":"Wed, 01 Jan 2014 09:08:45","bookmarklet":0,"mobile":0,"feed-item":"","from-feed-id":0,"unix-timestamp":1388585325,"format":"html","reblog-key":"jPfWHFnT","slug":"knock-knock","question":"Knock knock?","answer":"<p>Who is there?<\/p>"}]}'
+      @posts = JSON.parse(@jsonPayload)
+      @batch = @posts["posts"].map { |post| Importers::Tumblr.post_to_hash(post, 'html') }
+    end
 
-		should "have a post" do
-			assert_equal(1, @posts["posts"].size)
-	    end
+    should "have a post" do
+      assert_equal(2, @posts["posts"].size)
+    end
+  
+    should "convert post into hash" do
+      refute_nil(@batch, "a batch with a valid post should exist")
+    end
 
-	    should "have a regular post" do
-	    	assert_equal("regular", @posts['posts'][0]['type'])
-	    end
+    context "post" do
+      should "have a corresponding type" do
+        assert_equal("regular", @posts['posts'][0]['type'])
+      end
+  
+      should "have a hash with a valid name" do
+        assert_equal("2013-07-06-jekyll-test.html", @batch[0][:name])
+      end
+  
+      should "have a hash with a valid layout" do
+        assert_equal("post", @batch[0][:header]['layout'])
+      end
+  
+      should "have a hash with a valid title" do
+        assert_equal("Jekyll: Test", @batch[0][:header]['title'])
+      end
+  
+      should "have a hash with valid tags" do
+        assert_equal("jekyll", @batch[0][:header]['tags'][0])
+      end
+  
+      should "have a hash with valid content" do
+        assert_equal("<p>Testing...</p>", @batch[0][:content])
+      end
+  
+      should "have a hash with a valid url" do
+        assert_equal("https://github.com/post/54759400073", @batch[0][:url])
+      end
+  
+      should "have a hash with a valid slug" do
+        assert_equal("http://github.com/post/54759400073/jekyll-test", @batch[0][:slug])
+      end
+    end
 
-	    should "convert post into hash" do
-	    	batch = @posts["posts"].map { |post| Importers::Tumblr.post_to_hash(post, 'html') }
-	    	refute_nil(batch, "a batch with a valid post should exist")
-	    end
-
-	    should "have a hash with a valid name" do
-	    	batch = @posts["posts"].map { |post| Importers::Tumblr.post_to_hash(post, 'html') }
-	    	assert_equal("2013-07-06-jekyll-test.html", batch[0][:name])
-	    end
-
-	    should "have a hash with a valid layout" do
-	    	batch = @posts["posts"].map { |post| Importers::Tumblr.post_to_hash(post, 'html') }
-	    	assert_equal("post", batch[0][:header]['layout'])
-	    end
-
-	    should "have a hash with a valid title" do
-	    	batch = @posts["posts"].map { |post| Importers::Tumblr.post_to_hash(post, 'html') }
-	    	assert_equal("Jekyll: Test", batch[0][:header]['title'])
-	    end
-
-	    should "have a hash with valid tags" do
-	    	batch = @posts["posts"].map { |post| Importers::Tumblr.post_to_hash(post, 'html') }
-	    	assert_equal("jekyll", batch[0][:header]['tags'][0])
-	    end
-
-	    should "have a hash with valid content" do
-	    	batch = @posts["posts"].map { |post| Importers::Tumblr.post_to_hash(post, 'html') }
-	    	assert_equal("<p>Testing...</p>", batch[0][:content])
-	    end
-
-	    should "have a hash with a valid url" do
-	    	batch = @posts["posts"].map { |post| Importers::Tumblr.post_to_hash(post, 'html') }
-	    	assert_equal("https://github.com/post/54759400073", batch[0][:url])
-	    end
-
-	    should "have a hash with a valid slug" do
-	    	batch = @posts["posts"].map { |post| Importers::Tumblr.post_to_hash(post, 'html') }
-	    	assert_equal("http://github.com/post/54759400073/jekyll-test", batch[0][:slug])
-	    end
-	end
-end
+    context "answer" do
+      should "have a corresponding type" do
+        assert_equal("answer", @posts['posts'][1]['type'])
+      end
+  
+      should "have a hash with a valid name" do
+        assert_equal("2014-01-01-knock-knock.html", @batch[1][:name])
+      end
+  
+      should "have a hash with a valid layout" do
+        assert_equal("post", @batch[1][:header]['layout'])
+      end
+  
+      should "have a hash with a valid title" do
+        assert_equal("Knock knock?", @batch[1][:header]['title'])
+      end
+  
+      should "have a hash with valid tags" do
+        assert_equal([], @batch[1][:header]['tags'])
+      end
+  
+      should "have a hash with valid content" do
+        assert_equal("<p>Who is there?</p>", @batch[1][:content])
+      end
+  
+      should "have a hash with a valid url" do
+        assert_equal("http://example.com/post/71845593082", @batch[1][:url])
+      end
+  
+      should "have a hash with a valid slug" do
+        assert_equal("http://example.com/post/71845593082/knock-knock", @batch[1][:slug])
+      end
+    end
+  end
+end