jekyll-import 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84a8cd9c3d2ec9d745331b9152cb5cfb13e6c6dd
-  data.tar.gz: 897e5994d5344b11b717f7ec28db2cbc88bb8300
+  metadata.gz: 0b5d74f3eddb29a55ff845369991337d004ab36f
+  data.tar.gz: a06bcfad396653de2fd03b1146c3719ea83d73d1
 SHA512:
-  metadata.gz: e25bcb63ee3d7c082cadb08065d16cea5fa11f3790004156cc7584376e33f4d1fb6cbb70811a89b48f472065b255d9b86af4af0071b7dabe7b02ac9d4a121947
-  data.tar.gz: df0d984f949b75c495b1810cdfe1d0e6972a95d5842914fcd395fb8e53daf573218f8ccbd3cc2c6ef2c7f86d3b9f21b6539043b4e3952602b22b86f215479392
+  metadata.gz: e7eb1b3daf5ffd72d1127dee1298c998402987bd8eb254f293cd773b3bd5f12e2db11f12a3b97bc82cb33f54b6bdeb1842b14aac1b49b77cb91fba754e9a2567
+  data.tar.gz: b29b9ecafc01d25924932b73d5a48d272013e2523b7da7b94c4e298cc14321e108917a99f6918acedf4f89250fbffada82aa91e55af09d2769db940859533ae3

data/.travis.yml

@@ -1,10 +1,13 @@
 language: ruby
 before_install:
   - gem install bundler
+install:
+  - export NOKOGIRI_USE_SYSTEM_LIBRARIES=true
+  - travis_retry bundle install
 rvm:
-  - 2.0.0
+  - 2.1
+  - 2.0
   - 1.9.3
-  - 1.9.2
 script: bundle exec rake
 notifications:
   irc: "irc.freenode.org#jekyll"

data/History.markdown

@@ -10,6 +10,20 @@
 
 ### Development Fixes
 
+## 0.3.0 / 2014-05-23
+
+### Minor Enhancements
+
+  * Import WordPress.org `author` data as hash (#139)
+  * Add `socket` option to the WordPress importer (#140)
+  * Allow the CSV importer to skip writing front matter (#143)
+  * WordPress.com: Download images locally and update links to them (#134)
+  * WordPress: Import WP pages as proper Jekyll pages instead of as posts (#137)
+
+### Bug Fixes
+
+  * Replace errant `continue` expression with the valid `next` expression (#133)
+
 ## 0.2.0 / 2014-03-16
 
 ### Major Enhancements

data/README.markdown

@@ -1,18 +1,32 @@
-jekyll-import
-=============
+# jekyll-import
+
+[![Build Status](https://travis-ci.org/jekyll/jekyll-import.svg?branch=master)](https://travis-ci.org/jekyll/jekyll-import)
 
 The new __Jekyll__ command for importing from various blogs to Jekyll format.
 
 **Note: _migrators_ are now called _importers_ and are only available if one installs the `jekyll-import` _gem_.**
 
-How `jekyll-import` works:
----
+## How `jekyll-import` works:
+
+### Jekyll v2.x
+
+1. Install the _rubygem_ with `gem install jekyll-import`.
+2. Run `jekyll import IMPORTER [options]`
+
+### Jekyll v1.x
+
+Launch IRB:
 
-Install the _rubygem_ with `gem install jekyll-import`
-* In your IRB/Ruby configurations, `require jekyll-import`
-* For CLI uses, the newly installed CLI command is now a Jekyll plugin/sub-command invoked with `jekyll import`
+```ruby
+# 1. Require jekyll-import
+irb> require 'jekyll-import'
+# 2. Choose the importer you'd like to use.
+irb> importer_class = "Behance" # an example, there are many others!
+# 3. Run it!
+irb> JekyllImport::Importers.const_get(importer_class).run(options_hash)
+```
 
-Read [Blog migrations](http://jekyllrb.com/docs/migrations/) for more detailed info.
+## Documentation
 
-[![Build
-Status](https://travis-ci.org/jekyll/jekyll-import.png?branch=master)](https://travis-ci.org/jekyll/jekyll-import)
+jekyll-import has its own documentation site, found at http://import.jekyllrb.com.
+Dedicated [documentation for each migrator](http://import.jekyllrb.com/docs/home/) is available there.

data/jekyll-import.gemspec

@@ -40,7 +40,7 @@ Gem::Specification.new do |s|
 
   # test dependencies:
   s.add_development_dependency('redgreen', "~> 1.2")
-  s.add_development_dependency('shoulda', "~> 3.3.2")
+  s.add_development_dependency('shoulda', "~> 3.5")
   s.add_development_dependency('rr', "~> 1.0")
   s.add_development_dependency('simplecov', "~> 0.7")
   s.add_development_dependency('simplecov-gem-adapter', "~> 1.0.1")
@@ -53,6 +53,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency('pg', "~> 0.12")
   s.add_development_dependency('mysql2', "~> 0.3")
   s.add_development_dependency('behance', "~> 0.3")
+  s.add_development_dependency('unidecode')
 
   # site dependencies:
   s.add_development_dependency('launchy', '~> 2.4')

data/lib/jekyll-import/importers/csv.rb

@@ -10,6 +10,7 @@ module JekyllImport
 
       def self.specify_options(c)
         c.option 'file', '--file NAME', 'The CSV file to import (default: "posts.csv")'
+        c.option 'no-front-matter', '--no-front-matter', 'Do not add the default front matter to the post body'
       end
 
       # Reads a csv with title, permalink, body, published_at, and filter.
@@ -25,26 +26,33 @@ module JekyllImport
           next if row[0] == "title"
           posts += 1
           name = build_name(row)
-          write_post(name, row[0], row[2])
+          write_post(name, row[0], row[2], options)
         end
         "Created #{posts} posts!"
       end
 
-      def self.write_post(name, title, content)
+      def self.write_post(name, title, content, options)
         File.open("_posts/#{name}", "w") do |f|
+          write_frontmatter(f, title, options)
+          f.puts content
+        end
+      end
+
+      def self.build_name(row)
+        row[3].split(" ")[0]+"-"+row[1]+(row[4] =~ /markdown/ ? ".markdown" : ".textile")
+      end
+
+      def self.write_frontmatter(f, title, options)
+        no_frontmatter = options.fetch('no-front-matter', false)
+        unless no_frontmatter
           f.puts <<-HEADER
 ---
 layout: post
 title: #{title}
 ---
 HEADER
-          f.puts content
         end
       end
-
-      def self.build_name(row)
-        row[3].split(" ")[0]+"-"+row[1]+(row[4] =~ /markdown/ ? ".markdown" : ".textile")
-      end
     end
   end
 end

data/lib/jekyll-import/importers/google_reader.rb

@@ -1,7 +1,3 @@
-# Usage:
-#   (Local file)
-#   ruby -r 'jekyll/jekyll-import/rss' -e "JekyllImport::GoogleReader.process(:source => './somefile/on/your/computer.xml')"
-
 module JekyllImport
   module Importers
     class GoogleReader < Importer
@@ -21,7 +17,7 @@ module JekyllImport
           rss
           fileutils
           safe_yaml
-          open-url
+          open-uri
           rexml/document
           date
         ])

data/lib/jekyll-import/importers/wordpress.rb

@@ -17,11 +17,13 @@ module JekyllImport
           sequel
           fileutils
           safe_yaml
+          unidecode
         ])
       end
 
       def self.specify_options(c)
         c.option 'dbname', '--dbname DB', 'Database name (default: "")'
+        c.option 'socket', '--socket SOCKET', 'Database socket (default: "")'
         c.option 'user', '--user USER', 'Database user name (default: "")'
         c.option 'password', '--password PW', "Database user's password (default: "")"
         c.option 'host', '--host HOST', 'Database host name (default: "localhost")'
@@ -41,6 +43,7 @@ module JekyllImport
       # user::    The database user name
       # pass::    The database user's password
       # host::    The address of the MySQL database host. Default: 'localhost'
+      # socket::  The database socket's path
       # options:: A hash table of configuration options.
       #
       # Supported options are:
@@ -78,6 +81,7 @@ module JekyllImport
           :user           => opts.fetch('user', ''),
           :pass           => opts.fetch('password', ''),
           :host           => opts.fetch('host', 'localhost'),
+          :socket         => opts.fetch('socket', ''),
           :dbname         => opts.fetch('dbname', ''),
           :table_prefix   => opts.fetch('prefix', 'wp_'),
           :clean_entities => opts.fetch('clean_entities', true),
@@ -102,10 +106,31 @@ module JekyllImport
         FileUtils.mkdir_p("_posts")
 
         db = Sequel.mysql2(options[:dbname], :user => options[:user], :password => options[:pass],
-                          :host => options[:host], :encoding => 'utf8')
+                          :socket => options[:socket], :host => options[:host], :encoding => 'utf8')
 
         px = options[:table_prefix]
 
+        page_name_list = {}
+
+        page_name_query = "
+           SELECT
+             posts.ID            AS `id`,
+             posts.post_title    AS `title`,
+             posts.post_name     AS `slug`,
+             posts.post_parent   AS `parent`
+           FROM #{px}posts AS `posts`
+           WHERE posts.post_type = 'page'"
+
+        db[page_name_query].each do |page|
+          if !page[:slug] or page[:slug].empty?
+            page[:slug] = sluggify(page[:title])
+          end
+          page_name_list[ page[:id] ] = {
+            :slug   => page[:slug],
+            :parent => page[:parent]
+          }
+        end
+
         posts_query = "
            SELECT
              posts.ID            AS `id`,
@@ -138,12 +163,12 @@ module JekyllImport
         end
 
         db[posts_query].each do |post|
-          process_post(post, db, options)
+          process_post(post, db, options, page_name_list)
         end
       end
 
 
-      def self.process_post(post, db, options)
+      def self.process_post(post, db, options, page_name_list)
         px = options[:table_prefix]
 
         title = post[:title]
@@ -269,7 +294,12 @@ module JekyllImport
           'status'        => post[:status].to_s,
           'published'     => (post[:status].to_s == "publish"),
           'title'         => title.to_s,
-          'author'        => post[:author].to_s,
+          'author'        => {
+            'display_name'=> post[:author].to_s,
+            'login'       => post[:author_login].to_s,
+            'email'       => post[:author_email].to_s,
+            'url'  => post[:author_url].to_s,
+          },
           'author_login'  => post[:author_login].to_s,
           'author_email'  => post[:author_email].to_s,
           'author_url'    => post[:author_url].to_s,
@@ -284,8 +314,15 @@ module JekyllImport
           'comments'      => options[:comments] ? comments : nil,
         }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
 
+        if post[:type] == 'page'
+          filename = page_path(post[:id], page_name_list) + 'index.markdown'
+          FileUtils.mkdir_p(File.dirname(filename))
+        else
+          filename = "_posts/#{name}"
+        end
+
         # Write out the data and content to file
-        File.open("_posts/#{name}", "w") do |f|
+        File.open(filename, "w") do |f|
           f.puts data
           f.puts "---"
           f.puts Util.wpautop(content)
@@ -311,13 +348,19 @@ module JekyllImport
 
 
       def self.sluggify( title )
-        begin
-          require 'unidecode'
-          title = title.to_ascii
-        rescue LoadError
-          STDERR.puts "Could not require 'unidecode'. If your post titles have non-ASCII characters, you could get nicer permalinks by installing unidecode."
+        title = title.to_ascii.downcase.gsub(/[^0-9A-Za-z]+/, " ").strip.gsub(" ", "-")
+      end
+
+      def self.page_path( page_id, page_name_list )
+        if page_name_list.key?(page_id)
+          [
+            page_path(page_name_list[page_id][:parent],page_name_list),
+            page_name_list[page_id][:slug],
+            '/'
+          ].join("")
+        else
+          ""
         end
-        title.downcase.gsub(/[^0-9A-Za-z]+/, " ").strip.gsub(" ", "-")
       end
 
     end

data/lib/jekyll-import/importers/wordpressdotcom.rb

@@ -13,19 +13,54 @@ module JekyllImport
           safe_yaml
           hpricot
           time
+          open-uri
         ])
       end
 
       def self.specify_options(c)
         c.option 'source', '--source FILE', 'WordPress export XML file (default: "wordpress.xml")'
+        c.option 'no_fetch_images', '--no-fetch-images', 'Do not fetch the images referenced in the posts'
+        c.option 'assets_folder', '--assets_folder FOLDER', 'Folder where assets such as images will be downloaded to (default: assets)'
+      end
+
+      # Will modify post DOM tree
+      def self.download_images(title, post_hpricot, assets_folder)
+        images = (post_hpricot/"img")
+        if images.length == 0
+          return
+        end
+        puts "Downloading images for " + title
+        images.each do |i|
+          uri = i["src"]
+
+          i["src"] = assets_folder + "/" + File.basename(uri)
+          dst = File.join(assets_folder, File.basename(uri))
+          puts "  " + uri
+          if File.exist?(dst)
+            puts "    Already in cache. Clean assets folder if you want a redownload."
+            next
+          end
+          begin
+            open(uri) {|f|
+              File.open(dst, "wb") do |out|
+                out.puts f.read
+              end
+            }
+            puts "    OK!"
+          rescue => e
+            puts "    Errorr: #{e.message}"
+          end
+        end
       end
 
       def self.process(options)
-        source = options.fetch('source', "wordpress.xml")
+        source        = options.fetch('source', "wordpress.xml")
+        fetch         = !options.fetch('no_fetch_images', false)
+        assets_folder = options.fetch('assets_folder', 'assets')
+        FileUtils.mkdir_p(assets_folder)
 
         import_count = Hash.new(0)
         doc = Hpricot::XML(File.read(source))
-
         # Fetch authors data from header
         authors = Hash[
           (doc/:channel/'wp:author').map do |author|
@@ -83,11 +118,17 @@ module JekyllImport
           }
 
           begin
+            content = Hpricot(item.at('content:encoded').inner_text)
+
+            if fetch
+              download_images(title, content, assets_folder)
+            end
+
             FileUtils.mkdir_p "_#{type}s"
             File.open("_#{type}s/#{name}", "w") do |f|
               f.puts header.to_yaml
               f.puts '---'
-              f.puts Util.wpautop(item.at('content:encoded').inner_text)
+              f.puts Util.wpautop(content.to_html)
             end
           rescue => e
             puts "Couldn't import post!"

data/lib/jekyll-import/util.rb

@@ -27,7 +27,7 @@ module JekyllImport
 
           unless start
             pee += pee_part
-            continue
+            next
           end
 
           name = "<pre wp-pre-tag-#{i}></pre>"
@@ -73,4 +73,4 @@ module JekyllImport
       pee
     end
   end
-end
+end

data/lib/jekyll-import/version.rb

@@ -1,3 +1,3 @@
 module JekyllImport
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/site/_config.yml

@@ -4,4 +4,4 @@ relative_permalinks: false
 permalink: /news/:year/:month/:day/:title/
 excerpt_separator: ""
 repository: https://github.com/jekyll/jekyll-import
-latest_release: 0.1.0.beta4
+latest_release: 0.2.0

data/site/_posts/2014-05-23-jekyll-import-0-3-0-released.markdown

@@ -0,0 +1,22 @@
+---
+layout: news_item
+title: 'jekyll-import 0.3.0 Released'
+date: 2014-05-23 23:06:43 -0400
+author: parkr
+version: 0.3.0
+categories: [release]
+---
+
+Hey! This release is just to get a few fixes that have lain stale for a bit
+out into the open. Notably, the WordPress and WordPress.com importers have
+been markedly improved. Check out the full changelog for more details
+
+Highlights:
+
+* Import WordPress.org `author` data as hash
+* Add `socket` option to the WordPress importer
+* Allow the CSV importer to skip writing front matter
+* WordPress.com: Download images locally and update links to them
+* WordPress: Import WP pages as proper Jekyll pages instead of as posts
+
+Happy Jekylling!

data/site/docs/csv.md

@@ -23,3 +23,15 @@ Your file CSV file will be read in with the following columns:
 3. body
 4. published_at
 5. filter (e.g. markdown, textile)
+
+If you wish to specify custom front matter for each of your posts, you
+can use the `no-front-matter` option to prevent the default front matter
+from being written to the imported files:
+
+{% highlight bash %}
+$ ruby -rubygems -e 'require "jekyll-import";
+    JekyllImport::Importers::CSV.run({
+      "file" => "my_posts.csv",
+      "no-front-matter" => true
+    })'
+{% endhighlight %}

data/site/docs/history.md

@@ -1,10 +1,37 @@
 ---
 layout: docs
 title: History
-permalink: /docs/history/
+permalink: "/docs/history/"
 prev_section: contributing
 ---
 
+## 0.2.0 / 2014-03-16
+
+### Major Enhancements
+- Add comments to MovableType importer ([#66]({{ site.repository }}/issues/66))
+- Support auto-paragraph Wordpress convention ([#125]({{ site.repository }}/issues/125))
+
+### Minor Enhancements
+- Extract author info from wordpress XML files ([#131]({{ site.repository }}/issues/131))
+
+### Bug Fixes
+- Require YAML library in Enki importer ([#112]({{ site.repository }}/issues/112))
+- Fix !ruby/string:Sequel::SQL::Blob error in MT excerpts ([#113]({{ site.repository }}/issues/113))
+- Drupal6: Ensure post "tags" is "", never nil ([#117]({{ site.repository }}/issues/117))
+- Fixes a bug where the Tumblr importer would write posts with `nil`
+    content ([#118]({{ site.repository }}/issues/118))
+- Change Drupal7 layout to `post` rather than `default` ([#124]({{ site.repository }}/issues/124))
+- WordPress: Use explicit `#to_s` when outputting post dates ([#129]({{ site.repository }}/issues/129))
+
+### Site Enhancements
+- Fixed error in quickstart code, added required design changes/reflow ([#120]({{ site.repository }}/issues/120))
+- Fix example parameter names for Tumblr importer ([#122]({{ site.repository }}/issues/122))
+- Add note to WordPress installer docs page that indicates what the importer
+    does and doesn't do. ([#127]({{ site.repository }}/issues/127))
+
+### Development Fixes
+- Bring gemspec into the 2010's ([#130]({{ site.repository }}/issues/130))
+
 ## 0.1.0 / 2013-12-18
 
 ### Major Enhancements

data/site/docs/wordpress.md

@@ -17,6 +17,7 @@ $ ruby -rubygems -e 'require "jekyll-import";
       "user"     => "",
       "password" => "",
       "host"     => "localhost",
+      "socket"   => "",
       "prefix"   => "wp_",
       "clean_entities" => true,
       "comments"       => true,

data/site/docs/wordpressdotcom.md

@@ -12,11 +12,14 @@ To import your posts from a [WordPress.com](http://wordpress.com) blog, run:
 {% highlight bash %}
 $ ruby -rubygems -e 'require "jekyll-import";
     JekyllImport::Importers::WordpressDotCom.run({
-      "source" => "wordpress.xml"
+      "source" => "wordpress.xml",
+      "no_fetch_images" => false,
+      "assets_folder" => "assets"
     })'
 {% endhighlight %}
 
-The `source` field is not required. Its default is what you see above.
+The `source`, `no_fetch_images`, and `assets_folder` fields are not required.
+Their default values are what you see above.
 
 <div class="note">
   <h5>ProTip™: WordPress.com Export Tool</h5>

data/test/helper.rb

@@ -1,11 +1,10 @@
-if RUBY_VERSION > '1.9' && ENV["COVERAGE"] == "true"
+if ENV["COVERAGE"] == "true"
   require 'simplecov'
   require 'simplecov-gem-adapter'
   SimpleCov.start('gem')
 end
 
 require 'test/unit'
-require 'redgreen' if RUBY_VERSION < '1.9'
 require 'shoulda'
 require 'rr'
 
@@ -15,14 +14,14 @@ unless defined?(Test::Unit::AssertionFailedError)
   end
 end
 
-Dir.glob(File.expand_path('../../lib/*.rb', __FILE__)).each do |f|
-  require f
-end
+require File.expand_path('../../lib/jekyll-import.rb', __FILE__)
 
 include JekyllImport
 
+JekyllImport::Importer.subclasses.each(&:require_deps)
+
 # Send STDERR into the void to suppress program output messages
-STDERR.reopen(test(?e, '/dev/null') ? '/dev/null' : 'NUL:')
+#STDERR.reopen(test(?e, '/dev/null') ? '/dev/null' : 'NUL:')
 
 class Test::Unit::TestCase
   include RR::Adapters::TestUnit

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-import
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Tom Preston-Werner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-16 00:00:00.000000000 Z
+date: 2014-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jekyll
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.2
+        version: '3.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.2
+        version: '3.5'
 - !ruby/object:Gem::Dependency
   name: rr
   requirement: !ruby/object:Gem::Requirement
@@ -262,6 +262,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: unidecode
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: launchy
   requirement: !ruby/object:Gem::Requirement
@@ -343,6 +357,7 @@ files:
 - site/_posts/2013-11-18-jekyll-import-0-1-0-rc1-released.markdown
 - site/_posts/2013-12-17-jekyll-import-0-1-0-released.markdown
 - site/_posts/2014-03-16-jekyll-import-0-2-0-released.markdown
+- site/_posts/2014-05-23-jekyll-import-0-3-0-released.markdown
 - site/css/gridism.css
 - site/css/normalize.css
 - site/css/pygments.css