jekyll-import 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 73973a4031143ccdeea68fe8cabfc7ca023b4e8b
-  data.tar.gz: dd7ae5ee74214be7467c7a2ca0ebd92ef2db8178
+  metadata.gz: 84a8cd9c3d2ec9d745331b9152cb5cfb13e6c6dd
+  data.tar.gz: 897e5994d5344b11b717f7ec28db2cbc88bb8300
 SHA512:
-  metadata.gz: fe41f15390fec872bd74c8f232f3b2d2c2f355367dbbfc741e8d75351ae46352f3ddd18cbddd35cfca47cd01ac0ecb9880de1f99d3de371f34aa690b9e88cc6f
-  data.tar.gz: 9d97135a8616376d69646f72ca81a80f1c452964c681e2722e0ff57bc564e74af445803c2479a52550254215a93e078016724cf574fe150d5e23ab5260aee886
+  metadata.gz: e25bcb63ee3d7c082cadb08065d16cea5fa11f3790004156cc7584376e33f4d1fb6cbb70811a89b48f472065b255d9b86af4af0071b7dabe7b02ac9d4a121947
+  data.tar.gz: df0d984f949b75c495b1810cdfe1d0e6972a95d5842914fcd395fb8e53daf573218f8ccbd3cc2c6ef2c7f86d3b9f21b6539043b4e3952602b22b86f215479392

data/.gitignore

@@ -0,0 +1,21 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+
+.DS_Store
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+before_install:
+  - gem install bundler
+rvm:
+  - 2.0.0
+  - 1.9.3
+  - 1.9.2
+script: bundle exec rake
+notifications:
+  irc: "irc.freenode.org#jekyll"
+  email:
+    on_success: never
+    on_failure: never

data/History.markdown

@@ -10,6 +10,33 @@
 
 ### Development Fixes
 
+## 0.2.0 / 2014-03-16
+
+### Major Enhancements
+  * Add comments to MovableType importer (#66)
+  * Support auto-paragraph Wordpress convention (#125)
+
+### Minor Enhancements
+  * Extract author info from wordpress XML files (#131)
+
+### Bug Fixes
+  * Require YAML library in Enki importer (#112)
+  * Fix !ruby/string:Sequel::SQL::Blob error in MT excerpts (#113)
+  * Drupal6: Ensure post "tags" is "", never nil (#117)
+  * Fixes a bug where the Tumblr importer would write posts with `nil`
+    content (#118)
+  * Change Drupal7 layout to `post` rather than `default` (#124)
+  * WordPress: Use explicit `#to_s` when outputting post dates (#129)
+
+### Site Enhancements
+  * Fixed error in quickstart code, added required design changes/reflow (#120)
+  * Fix example parameter names for Tumblr importer (#122)
+  * Add note to WordPress installer docs page that indicates what the importer
+    does and doesn't do. (#127)
+
+### Development Fixes
+  * Bring gemspec into the 2010's (#130)
+
 ## 0.1.0 / 2013-12-18
 
 ### Major Enhancements

data/Rakefile

@@ -1,12 +1,5 @@
 # require 'rake/testtask'
 
-# Rake::TestTask.new do |t|
-#   t.libs << 'test'
-# end
-
-# desc "Run tests"
-# task :default => :test
-
 require 'rubygems'
 require 'rake'
 require 'rdoc'
@@ -24,16 +17,7 @@ def name
 end
 
 def version
-  line = File.read("lib/#{name}.rb")[/^\s*VERSION\s*=\s*.*/]
-  line.match(/.*VERSION\s*=\s*['"](.*)['"]/)[1]
-end
-
-def date
-  Date.today.to_s
-end
-
-def rubyforge_project
-  name
+  JekyllImport::VERSION
 end
 
 def gemspec_file
@@ -44,10 +28,6 @@ def gem_file
   "#{name}-#{version}.gem"
 end
 
-def replace_header(head, header_name)
-  head.sub!(/(\.#{header_name}\s*= ').*'/) { "#{$1}#{send(header_name)}'"}
-end
-
 def normalize_bullets(markdown)
   markdown.gsub(/\s{2}\*{1}/, "-")
 end
@@ -165,8 +145,12 @@ namespace :site do
           file.write(converted_history(history_file))
         end
       end
-      sh "git add site/docs/history.md"
-      sh "git commit -m 'Updated generated history.md file in the site.'"
+      unless `git diff site/docs/history.md`.strip.empty?
+        sh "git add site/docs/history.md"
+        sh "git commit -m 'Updated generated history.md file in the site.'"
+      else
+        puts "No updates to commit at this time. Skipping..."
+      end
     else
       abort "You seem to have misplaced your History.markdown file. I can haz?"
     end
@@ -218,37 +202,8 @@ task :release => :build do
 end
 
 desc "Build #{gem_file} into the pkg directory"
-task :build => :gemspec do
+task :build do
   sh "mkdir -p pkg"
   sh "gem build #{gemspec_file}"
   sh "mv #{gem_file} pkg"
 end
-
-desc "Generate #{gemspec_file}"
-task :gemspec do
-  # read spec file and split out manifest section
-  spec = File.read(gemspec_file)
-  head, manifest, tail = spec.split("  # = MANIFEST =\n")
-
-  # replace name version and date
-  replace_header(head, :name)
-  replace_header(head, :version)
-  replace_header(head, :date)
-  #comment this out if your rubyforge_project has a different name
-  replace_header(head, :rubyforge_project)
-
-  # determine file list from git ls-files
-  files = `git ls-files`.
-    split("\n").
-    sort.
-    reject { |file| file =~ /^\./ }.
-    reject { |file| file =~ /^(rdoc|pkg)/ }.
-    map { |file| "    #{file}" }.
-    join("\n")
-
-  # piece file back together and write
-  manifest = "  s.files = %w[\n#{files}\n  ]\n"
-  spec = [head, manifest, tail].join("  # = MANIFEST =\n")
-  File.open(gemspec_file, 'w') { |io| io.write(spec) }
-  puts "Updated #{gemspec_file}"
-end

data/jekyll-import.gemspec

@@ -1,13 +1,17 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'jekyll-import/version'
+
 Gem::Specification.new do |s|
   s.specification_version = 2 if s.respond_to? :specification_version=
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.rubygems_version = '1.3.5'
-  s.required_ruby_version = '>= 1.9.2'
+  s.rubygems_version = '2.2.2'
+  s.required_ruby_version = '>= 1.9.3'
 
-  s.name              = 'jekyll-import'
-  s.version           = '0.1.0'
-  s.date              = '2013-12-18'
-  s.rubyforge_project = 'jekyll-import'
+  s.name    = 'jekyll-import'
+  s.version = JekyllImport::VERSION
+  s.license = 'MIT'
 
   s.summary     = "Import command for Jekyll (static site generator)."
   s.description = "Provides the Import command for Jekyll."
@@ -15,17 +19,19 @@ Gem::Specification.new do |s|
   s.authors  = ["Tom Preston-Werner"]
   s.email    = 'tom@mojombo.com'
   s.homepage = 'http://github.com/jekyll/jekyll-import'
-  s.license  = 'MIT'
 
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.require_paths = %w[lib]
 
   s.rdoc_options = ["--charset=UTF-8"]
   s.extra_rdoc_files = %w[README.markdown LICENSE]
 
-  s.add_runtime_dependency('jekyll', '~> 1.3')
+  # runtime dependencies
+  s.add_runtime_dependency('jekyll', '~> 1.4')
   s.add_runtime_dependency('fastercsv')
   s.add_runtime_dependency('nokogiri')
-  s.add_runtime_dependency('safe_yaml', '~> 0.9.7')
 
   # development dependencies
   s.add_development_dependency('rake', "~> 10.1.0")
@@ -46,115 +52,8 @@ Gem::Specification.new do |s|
   s.add_development_dependency('mysql', "~> 2.8")
   s.add_development_dependency('pg', "~> 0.12")
   s.add_development_dependency('mysql2', "~> 0.3")
-  s.add_development_dependency('behance', "~> 0.3.0")
-
-  # = MANIFEST =
-  s.files = %w[
-    Gemfile
-    History.markdown
-    LICENSE
-    README.markdown
-    Rakefile
-    jekyll-import.gemspec
-    lib/jekyll-import.rb
-    lib/jekyll-import/importer.rb
-    lib/jekyll-import/importers.rb
-    lib/jekyll-import/importers/behance.rb
-    lib/jekyll-import/importers/csv.rb
-    lib/jekyll-import/importers/drupal6.rb
-    lib/jekyll-import/importers/drupal7.rb
-    lib/jekyll-import/importers/enki.rb
-    lib/jekyll-import/importers/ghost.rb
-    lib/jekyll-import/importers/google_reader.rb
-    lib/jekyll-import/importers/joomla.rb
-    lib/jekyll-import/importers/jrnl.rb
-    lib/jekyll-import/importers/marley.rb
-    lib/jekyll-import/importers/mephisto.rb
-    lib/jekyll-import/importers/mt.rb
-    lib/jekyll-import/importers/posterous.rb
-    lib/jekyll-import/importers/rss.rb
-    lib/jekyll-import/importers/s9y.rb
-    lib/jekyll-import/importers/textpattern.rb
-    lib/jekyll-import/importers/tumblr.rb
-    lib/jekyll-import/importers/typo.rb
-    lib/jekyll-import/importers/wordpress.rb
-    lib/jekyll-import/importers/wordpressdotcom.rb
-    lib/jekyll/commands/import.rb
-    site/.gitignore
-    site/CNAME
-    site/README
-    site/_config.yml
-    site/_includes/analytics.html
-    site/_includes/docs_contents.html
-    site/_includes/docs_contents_mobile.html
-    site/_includes/docs_option.html
-    site/_includes/docs_ul.html
-    site/_includes/footer.html
-    site/_includes/header.html
-    site/_includes/news_contents.html
-    site/_includes/news_contents_mobile.html
-    site/_includes/news_item.html
-    site/_includes/primary-nav-items.html
-    site/_includes/section_nav.html
-    site/_includes/top.html
-    site/_layouts/default.html
-    site/_layouts/docs.html
-    site/_layouts/news.html
-    site/_layouts/news_item.html
-    site/_posts/2013-11-09-jekyll-import-0-1-0-beta4-release.markdown
-    site/_posts/2013-11-18-jekyll-import-0-1-0-rc1-released.markdown
-    site/_posts/2013-12-17-jekyll-import-0-1-0-released.markdown
-    site/css/gridism.css
-    site/css/normalize.css
-    site/css/pygments.css
-    site/css/style.css
-    site/docs/behance.md
-    site/docs/contributing.md
-    site/docs/csv.md
-    site/docs/drupal6.md
-    site/docs/drupal7.md
-    site/docs/enki.md
-    site/docs/ghost.md
-    site/docs/google_reader.md
-    site/docs/history.md
-    site/docs/index.md
-    site/docs/installation.md
-    site/docs/joomla.md
-    site/docs/jrnl.md
-    site/docs/marley.md
-    site/docs/mephisto.md
-    site/docs/mt.md
-    site/docs/posterous.md
-    site/docs/rss.md
-    site/docs/s9y.md
-    site/docs/textpattern.md
-    site/docs/third-party.md
-    site/docs/tumblr.md
-    site/docs/typo.md
-    site/docs/usage.md
-    site/docs/wordpress.md
-    site/docs/wordpressdotcom.md
-    site/favicon.png
-    site/feed.xml
-    site/img/article-footer.png
-    site/img/footer-arrow.png
-    site/img/footer-logo.png
-    site/img/logo-2x.png
-    site/img/octojekyll.png
-    site/img/tube.png
-    site/img/tube1x.png
-    site/index.html
-    site/js/modernizr-2.5.3.min.js
-    site/news/index.html
-    site/news/releases/index.html
-    test/helper.rb
-    test/test_jrnl_importer.rb
-    test/test_mt_importer.rb
-    test/test_tumblr_importer.rb
-    test/test_wordpress_importer.rb
-    test/test_wordpressdotcom_importer.rb
-  ]
-  # = MANIFEST =
+  s.add_development_dependency('behance', "~> 0.3")
 
-  s.test_files = s.files.select { |path| path =~ /^test\/test_.*\.rb/ }
+  # site dependencies:
+  s.add_development_dependency('launchy', '~> 2.4')
 end

data/lib/jekyll-import.rb

@@ -6,10 +6,9 @@ require 'colorator'
 
 require 'jekyll-import/importer'
 require 'jekyll-import/importers'
+require 'jekyll-import/util'
 
 module JekyllImport
-  VERSION = '0.1.0'
-
   def self.logger
     @logger ||= Jekyll::Stevenson.new
   end
@@ -25,7 +24,7 @@ module JekyllImport
       name = importer.to_s.split("::").last.downcase
       commands << name
       cmd.command(name.to_sym) do |c|
-        c.syntax "jekyll import #{name} [options]"
+        c.syntax "#{name} [options]"
         importer.specify_options(c)
         c.action do |_, options|
           importer.run(options)

data/lib/jekyll-import/importer.rb

@@ -17,10 +17,10 @@ module JekyllImport
     end
 
     def self.run(options = {})
-      options = stringify_keys(options)
+      opts = stringify_keys(options)
       self.require_deps
-      self.validate(options) if self.respond_to?(:validate)
-      self.process(options)
+      self.validate(opts) if self.respond_to?(:validate)
+      self.process(opts)
     end
   end
 end

data/lib/jekyll-import/importers/drupal6.rb

@@ -87,7 +87,7 @@ EOF
           node_id = post[:nid]
           title = post[:title]
           content = post[:body]
-          tags = post.fetch(:tags, '').downcase.strip
+          tags = (post[:tags] || '').downcase.strip
           created = post[:created]
           time = Time.at(created)
           is_published = post[:status] == 1

data/lib/jekyll-import/importers/drupal7.rb

@@ -78,7 +78,7 @@ module JekyllImport
           # Get the relevant fields as a hash, delete empty fields and convert
           # to YAML for the header
           data = {
-             'layout' => 'default',
+             'layout' => 'post',
              'title' => title.to_s,
              'created' => created,
            }.delete_if { |k,v| v.nil? || v == ''}.to_yaml

data/lib/jekyll-import/importers/enki.rb

@@ -35,6 +35,7 @@ EOS
           sequel
           fileutils
           pg
+          yaml
         ])
       end
 

data/lib/jekyll-import/importers/mt.rb

@@ -18,10 +18,11 @@ module JekyllImport
 
       def self.default_options
         {
-          :blog_id => nil,
-          :categories => true,
-          :dest_encoding => 'utf-8',
-          :src_encoding => 'utf-8'
+          'blog_id' => nil,
+          'categories' => true,
+          'dest_encoding' => 'utf-8',
+          'src_encoding' => 'utf-8',
+          'comments' => false
         }
       end
 
@@ -54,21 +55,26 @@ module JekyllImport
       #
       # Supported options are:
       #
-      # :blog_id::        Specify a single MovableType blog to export by providing blog_id.
+      # blog_id::         Specify a single MovableType blog to export by providing blog_id.
       #                   Default: nil, importer will include posts for all blogs.
-      # :categories::     If true, save the post's categories in its
+      # categories::      If true, save the post's categories in its
       #                   YAML front matter. Default: true
-      # :src_encoding::   Encoding of strings from the database. Default: UTF-8
+      # src_encoding::    Encoding of strings from the database. Default: UTF-8
       #                   If your output contains mangled characters, set src_encoding to
       #                   something appropriate for your database charset.
-      # :dest_encoding::  Encoding of output strings. Default: UTF-8
+      # dest_encoding::   Encoding of output strings. Default: UTF-8
+      # comments::        If true, output comments in _comments directory, like the one
+      #                   mentioned at https://github.com/mpalmer/jekyll-static-comments/
       def self.process(options)
-        dbname = options.fetch('dbname')
-        user   = options.fetch('user')
-        pass   = options.fetch('password', "")
-        host   = options.fetch('host', "localhost")
+        options  = default_options.merge(options)
 
-        options = default_options.merge(options)
+        dbname   = options.fetch('dbname')
+        user     = options.fetch('user')
+        pass     = options.fetch('password', "")
+        host     = options.fetch('host', "localhost")
+        comments = options.fetch('comments')
+
+        posts_name_by_id = { } if comments
 
         db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host)
         post_categories = db[:mt_placement].join(:mt_category, :category_id => :placement_category_id)
@@ -76,7 +82,7 @@ module JekyllImport
         FileUtils.mkdir_p "_posts"
 
         posts = db[:mt_entry]
-        posts = posts.filter(:entry_blog_id => options[:blog_id]) if options[:blog_id]
+        posts = posts.filter(:entry_blog_id => options['blog_id']) if options['blog_id']
         posts.each do |post|
           categories = post_categories.filter(
             :mt_placement__placement_entry_id => post[:entry_id]
@@ -85,9 +91,12 @@ module JekyllImport
           file_name = post_file_name(post, options)
 
           data = post_metadata(post, options)
-          data['categories'] = categories if !categories.empty? && options[:categories]
+          data['categories'] = categories if !categories.empty? && options['categories']
           yaml_front_matter = data.delete_if { |k,v| v.nil? || v == '' }.to_yaml
 
+          # save post path for comment processing
+          posts_name_by_id[data['post_id']] = file_name if comments
+
           content = post_content(post, options)
 
           File.open("_posts/#{file_name}", "w") do |f|
@@ -96,6 +105,30 @@ module JekyllImport
             f.puts encode(content, options)
           end
         end
+
+        # process comment output, if enabled
+        if comments
+          FileUtils.mkdir_p "_comments"
+
+          comments = db[:mt_comment]
+          comments.each do |comment|
+            if posts_name_by_id.has_key?(comment[:comment_entry_id]) # if the entry exists
+              dir_name, base_name = comment_file_dir_and_base_name(posts_name_by_id, comment, options)
+              FileUtils.mkdir_p "_comments/#{dir_name}"
+
+              data = comment_metadata(comment, options)
+              content = comment_content(comment, options)
+              yaml_front_matter = data.delete_if { |k,v| v.nil? || v == '' }.to_yaml
+
+              File.open("_comments/#{dir_name}/#{base_name}", "w") do |f|
+                f.puts yaml_front_matter
+                f.puts "---"
+                f.puts encode(content, options)
+              end
+            end
+          end
+        end
+
       end
 
       # Extracts metadata for YAML front matter from post
@@ -104,8 +137,11 @@ module JekyllImport
           'layout' => 'post',
           'title' => encode(post[:entry_title], options),
           'date' => post_date(post).strftime("%Y-%m-%d %H:%M:%S %z"),
-          'excerpt' => encode(post[:entry_excerpt], options),
-          'mt_id' => post[:entry_id]
+          'excerpt' => encode(post[:entry_excerpt].to_s, options),
+          'mt_id' => post[:entry_id],
+          'blog_id' => post[:entry_blog_id],
+          'post_id' => post[:entry_id], # for link with comments
+          'basename' => post[:entry_basename]
         }
         metadata['published'] = false if post[:entry_status] != STATUS_PUBLISHED
         metadata
@@ -137,9 +173,42 @@ module JekyllImport
         "#{date.strftime('%Y-%m-%d')}-#{slug}.#{file_ext}"
       end
 
+      # Extracts metadata for YAML front matter from comment
+      def self.comment_metadata(comment, options = default_options)
+        metadata = {
+          'layout' => 'comment',
+          'comment_id' => comment[:comment_id],
+          'post_id' => comment[:comment_entry_id],
+          'author' => encode(comment[:comment_author], options),
+          'email' => comment[:comment_email],
+          'commenter_id' => comment[:comment_commenter_id],
+          'date' => comment_date(comment).strftime("%Y-%m-%d %H:%M:%S %z"),
+          'visible' => comment[:comment_visible] == 1,
+          'ip' => comment[:comment_ip],
+          'url' => comment[:comment_url]
+        }
+        metadata
+      end
+
+      # Different versions of MT used different column names
+      def self.comment_date(comment)
+        comment[:comment_modified_on] || comment[:comment_created_on]
+      end
+
+      def self.comment_content(comment, options = default_options)
+        comment[:comment_text]
+      end
+
+      def self.comment_file_dir_and_base_name(posts_name_by_id, comment, options = default_options)
+        post_basename = posts_name_by_id[comment[:comment_entry_id]].sub(/\.\w+$/, '')
+        comment_id = comment[:comment_id]
+
+        [post_basename, "#{comment_id}.markdown"]
+      end
+
       def self.encode(str, options = default_options)
         if str.respond_to?(:encoding)
-          str.encode(options[:dest_encoding], options[:src_encoding])
+          str.encode(options['dest_encoding'], options['src_encoding'])
         else
           str
         end