jekyll 0.11.2 → 0.12.0

data/lib/jekyll/migrators/wordpress.rb

@@ -11,52 +11,284 @@ require 'yaml'
 
 module Jekyll
   module WordPress
-    def self.process(dbname, user, pass, host = 'localhost', table_prefix = 'wp_')
-      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+
+    # Main migrator function. Call this to perform the migration.
+    # 
+    # dbname::  The name of the database
+    # user::    The database user name
+    # pass::    The database user's password
+    # host::    The address of the MySQL database host. Default: 'localhost'
+    # options:: A hash table of configuration options.
+    # 
+    # Supported options are:
+    # 
+    # :table_prefix::   Prefix of database tables used by WordPress.
+    #                   Default: 'wp_'
+    # :clean_entities:: If true, convert non-ASCII characters to HTML
+    #                   entities in the posts, comments, titles, and
+    #                   names. Requires the 'htmlentities' gem to
+    #                   work. Default: true.
+    # :comments::       If true, migrate post comments too. Comments
+    #                   are saved in the post's YAML front matter.
+    #                   Default: true.
+    # :categories::     If true, save the post's categories in its
+    #                   YAML front matter.
+    # :tags::           If true, save the post's tags in its
+    #                   YAML front matter.
+    # :more_excerpt::   If true, when a post has no excerpt but
+    #                   does have a <!-- more --> tag, use the
+    #                   preceding post content as the excerpt.
+    #                   Default: true.
+    # :more_anchor::    If true, convert a <!-- more --> tag into
+    #                   two HTML anchors with ids "more" and
+    #                   "more-NNN" (where NNN is the post number).
+    #                   Default: true.
+    # :status::         Array of allowed post statuses. Only
+    #                   posts with matching status will be migrated.
+    #                   Known statuses are :publish, :draft, :private,
+    #                   and :revision. If this is nil or an empty
+    #                   array, all posts are migrated regardless of
+    #                   status. Default: [:publish].
+    # 
+    def self.process(dbname, user, pass, host='localhost', options={})
+      options = {
+        :table_prefix   => 'wp_',
+        :clean_entities => true,
+        :comments       => true,
+        :categories     => true,
+        :tags           => true,
+        :more_excerpt   => true,
+        :more_anchor    => true,
+        :status         => [:publish] # :draft, :private, :revision
+      }.merge(options)
+
+      if options[:clean_entities]
+        begin
+          require 'htmlentities'
+        rescue LoadError
+          STDERR.puts "Could not require 'htmlentities', so the " +
+                      ":clean_entities option is now disabled."
+          options[:clean_entities] = false
+        end
+      end
 
       FileUtils.mkdir_p("_posts")
 
-      # Reads a MySQL database via Sequel and creates a post file for each
-      # post in wp_posts that has post_status = 'publish'. This restriction is
-      # made because 'draft' posts are not guaranteed to have valid dates.
-      query = "SELECT post_title, \
-                      post_name, \
-                      post_date, \
-                      post_content, \
-                      post_excerpt, \
-                      ID, \
-                      guid \
-               FROM #{table_prefix}posts \
-               WHERE post_status = 'publish' AND \
-                     post_type = 'post'"
-
-      db[query].each do |post|
-        # Get required fields and construct Jekyll compatible name.
-        title = post[:post_title]
-        slug = post[:post_name]
-        date = post[:post_date]
-        content = post[:post_content]
-        name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day,
-                                               slug]
-
-        # Get the relevant fields as a hash, delete empty fields and convert
-        # to YAML for the header.
-        data = {
-           'layout' => 'post',
-           'title' => title.to_s,
-           'excerpt' => post[:post_excerpt].to_s,
-           'wordpress_id' => post[:ID],
-           'wordpress_url' => post[:guid],
-           'date' => date
-         }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
-
-        # Write out the data and content to file
-        File.open("_posts/#{name}", "w") do |f|
-          f.puts data
-          f.puts "---"
-          f.puts content
+      db = Sequel.mysql(dbname, :user => user, :password => pass,
+                        :host => host, :encoding => 'utf8')
+
+      px = options[:table_prefix]
+
+      posts_query = "
+         SELECT
+           posts.ID            AS `id`,
+           posts.guid          AS `guid`,
+           posts.post_type     AS `type`,
+           posts.post_status   AS `status`,
+           posts.post_title    AS `title`,
+           posts.post_name     AS `slug`,
+           posts.post_date     AS `date`,
+           posts.post_content  AS `content`,
+           posts.post_excerpt  AS `excerpt`,
+           posts.comment_count AS `comment_count`,
+           users.display_name  AS `author`,
+           users.user_login    AS `author_login`,
+           users.user_email    AS `author_email`,
+           users.user_url      AS `author_url`
+         FROM #{px}posts AS `posts`
+           LEFT JOIN #{px}users AS `users`
+             ON posts.post_author = users.ID"
+
+      if options[:status] and not options[:status].empty?
+        status = options[:status][0]
+        posts_query << "
+         WHERE posts.post_status = '#{status.to_s}'"
+        options[:status][1..-1].each do |status|
+          posts_query << " OR
+           posts.post_status = '#{status.to_s}'"
+        end
+      end
+
+      db[posts_query].each do |post|
+        process_post(post, db, options)
+      end
+    end
+
+
+    def self.process_post(post, db, options)
+      px = options[:table_prefix]
+
+      title = post[:title]
+      if options[:clean_entities]
+        title = clean_entities(title)
+      end
+
+      slug = post[:slug]
+      if !slug or slug.empty?
+        slug = sluggify(title)
+      end
+
+      date = post[:date] || Time.now
+      name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month,
+                                             date.day, slug]
+      content = post[:content].to_s
+      if options[:clean_entities]
+        content = clean_entities(content)
+      end
+
+      excerpt = post[:excerpt].to_s
+
+      more_index = content.index(/<!-- *more *-->/)
+      more_anchor = nil
+      if more_index
+        if options[:more_excerpt] and
+            (post[:excerpt].nil? or post[:excerpt].empty?)
+          excerpt = content[0...more_index]
+        end
+        if options[:more_anchor]
+          more_link = "more"
+          content.sub!(/<!-- *more *-->/,
+                       "<a id=\"more\"></a>" + 
+                       "<a id=\"more-#{post[:id]}\"></a>")
+        end
+      end
+
+      categories = []
+      tags = []
+
+      if options[:categories] or options[:tags]
+
+        cquery =
+          "SELECT
+             terms.name AS `name`,
+             ttax.taxonomy AS `type`
+           FROM
+             #{px}terms AS `terms`,
+             #{px}term_relationships AS `trels`,
+             #{px}term_taxonomy AS `ttax`
+           WHERE
+             trels.object_id = '#{post[:id]}' AND
+             trels.term_taxonomy_id = ttax.term_taxonomy_id AND
+             terms.term_id = ttax.term_id"
+
+        db[cquery].each do |term|
+          if options[:categories] and term[:type] == "category"
+            if options[:clean_entities]
+              categories << clean_entities(term[:name])
+            else
+              categories << term[:name]
+            end
+          elsif options[:tags] and term[:type] == "post_tag"
+            if options[:clean_entities]
+              tags << clean_entities(term[:name])
+            else
+              tags << term[:name]
+            end
+          end
         end
       end
+
+      comments = []
+
+      if options[:comments] and post[:comment_count].to_i > 0
+        cquery =
+          "SELECT
+             comment_ID           AS `id`,
+             comment_author       AS `author`,
+             comment_author_email AS `author_email`,
+             comment_author_url   AS `author_url`,
+             comment_date         AS `date`,
+             comment_date_gmt     AS `date_gmt`,
+             comment_content      AS `content`
+           FROM #{px}comments
+           WHERE
+             comment_post_ID = '#{post[:id]}' AND
+             comment_approved != 'spam'"
+
+
+        db[cquery].each do |comment|
+
+          comcontent = comment[:content].to_s
+          if comcontent.respond_to?(:force_encoding)
+            comcontent.force_encoding("UTF-8")
+          end
+          if options[:clean_entities]
+            comcontent = clean_entities(comcontent)
+          end
+          comauthor = comment[:author].to_s
+          if options[:clean_entities]
+            comauthor = clean_entities(comauthor)
+          end
+
+          comments << {
+            'id'           => comment[:id].to_i,
+            'author'       => comauthor,
+            'author_email' => comment[:author_email].to_s,
+            'author_url'   => comment[:author_url].to_s,
+            'date'         => comment[:date].to_s,
+            'date_gmt'     => comment[:date_gmt].to_s,
+            'content'      => comcontent,
+          }
+        end
+
+        comments.sort!{ |a,b| a['id'] <=> b['id'] }
+      end
+
+      # Get the relevant fields as a hash, delete empty fields and
+      # convert to YAML for the header.
+      data = {
+        'layout'        => post[:type].to_s,
+        'status'        => post[:status].to_s,
+        'published'     => (post[:status].to_s == "publish"),
+        'title'         => title.to_s,
+        'author'        => post[:author].to_s,
+        'author_login'  => post[:author_login].to_s,
+        'author_email'  => post[:author_email].to_s,
+        'author_url'    => post[:author_url].to_s,
+        'excerpt'       => excerpt,
+        'more_anchor'   => more_anchor,
+        'wordpress_id'  => post[:id],
+        'wordpress_url' => post[:guid].to_s,
+        'date'          => date,
+        'categories'    => options[:categories] ? categories : nil,
+        'tags'          => options[:tags] ? tags : nil,
+        'comments'      => options[:comments] ? comments : nil,
+      }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
+
+      # Write out the data and content to file
+      File.open("_posts/#{name}", "w") do |f|
+        f.puts data
+        f.puts "---"
+        f.puts content
+      end
+    end
+
+
+    def self.clean_entities( text )
+      if text.respond_to?(:force_encoding)
+        text.force_encoding("UTF-8")
+      end
+      text = HTMLEntities.new.encode(text, :named)
+      # We don't want to convert these, it would break all
+      # HTML tags in the post and comments.
+      text.gsub!("&amp;", "&")
+      text.gsub!("&lt;", "<")
+      text.gsub!("&gt;", ">")
+      text.gsub!("&quot;", '"')
+      text.gsub!("&apos;", "'")
+      text
+    end
+
+
+    def self.sluggify( title )
+      begin
+        require 'unidecode'
+        title = title.to_ascii
+      rescue LoadError
+        STDERR.puts "Could not require 'unidecode'. If your post titles have non-ASCII characters, you could get nicer permalinks by installing unidecode."
+      end
+      title.downcase.gsub(/[^0-9A-Za-z]+/, " ").strip.gsub(" ", "-")
     end
+
   end
 end

data/lib/jekyll/post.rb

@@ -22,6 +22,8 @@ module Jekyll
     attr_accessor :data, :content, :output, :ext
     attr_accessor :date, :slug, :published, :tags, :categories
 
+    attr_reader :name
+
     # Initialize this Post instance.
     #   +site+ is the Site
     #   +base+ is the String path to the dir containing the post file
@@ -36,7 +38,11 @@ module Jekyll
 
       self.categories = dir.split('/').reject { |x| x.empty? }
       self.process(name)
-      self.read_yaml(@base, name)
+      begin 
+        self.read_yaml(@base, name)
+      rescue Exception => msg
+        raise FatalException.new("#{msg} in #{@base}/#{name}")        
+      end
 
       #If we've added a date and time to the yaml, use that instead of the filename date
       #Means we'll sort correctly.
@@ -131,7 +137,7 @@ module Jekyll
           "title"      => CGI.escape(slug),
           "i_day"      => date.strftime("%d").to_i.to_s,
           "i_month"    => date.strftime("%m").to_i.to_s,
-          "categories" => categories.join('/'),
+          "categories" => categories.map { |c| URI.escape(c) }.join('/'),
           "output_ext" => self.output_ext
         }.inject(template) { |result, token|
           result.gsub(/:#{Regexp.escape token.first}/, token.last)

data/lib/jekyll/site.rb

@@ -4,7 +4,7 @@ module Jekyll
 
   class Site
     attr_accessor :config, :layouts, :posts, :pages, :static_files,
-                  :categories, :exclude, :source, :dest, :lsi, :pygments,
+                  :categories, :exclude, :include, :source, :dest, :lsi, :pygments,
                   :permalink_style, :tags, :time, :future, :safe, :plugins, :limit_posts
 
     attr_accessor :converters, :generators
@@ -18,11 +18,12 @@ module Jekyll
       self.safe            = config['safe']
       self.source          = File.expand_path(config['source'])
       self.dest            = File.expand_path(config['destination'])
-      self.plugins         = File.expand_path(config['plugins'])
+      self.plugins         = Array(config['plugins']).map { |d| File.expand_path(d) }
       self.lsi             = config['lsi']
       self.pygments        = config['pygments']
       self.permalink_style = config['permalink'].to_sym
       self.exclude         = config['exclude'] || []
+      self.include         = config['include'] || []
       self.future          = config['future']
       self.limit_posts     = config['limit_posts'] || nil
 
@@ -72,8 +73,10 @@ module Jekyll
       # If safe mode is off, load in any Ruby files under the plugins
       # directory.
       unless self.safe
-        Dir[File.join(self.plugins, "**/*.rb")].each do |f|
-          require f
+        self.plugins.each do |plugins|
+            Dir[File.join(plugins, "**/*.rb")].each do |f|
+              require f
+            end
         end
       end
 
@@ -98,12 +101,12 @@ module Jekyll
       self.read_directories
     end
 
-    # Read all the files in <source>/<dir>/_layouts and create a new Layout
-    # object with each one.
+    # Read all the files in <source>/<layouts> and create a new Layout object
+    # with each one.
     #
     # Returns nothing.
-    def read_layouts(dir = '')
-      base = File.join(self.source, dir, "_layouts")
+    def read_layouts
+      base = File.join(self.source, self.config['layouts'])
       return unless File.exists?(base)
       entries = []
       Dir.chdir(base) { entries = filter_entries(Dir['*.*']) }
@@ -118,12 +121,12 @@ module Jekyll
     # that will become part of the site according to the rules in
     # filter_entries.
     #
-    # dir - The String relative path of the directory to read.
+    # dir - The String relative path of the directory to read. Default: ''.
     #
     # Returns nothing.
     def read_directories(dir = '')
       base = File.join(self.source, dir)
-      entries = Dir.chdir(base) { filter_entries(Dir['*']) }
+      entries = Dir.chdir(base) { filter_entries(Dir.entries('.')) }
 
       self.read_posts(dir)
 
@@ -173,7 +176,10 @@ module Jekyll
       self.posts.sort!
 
       # limit the posts if :limit_posts option is set
-      self.posts = self.posts[-limit_posts, limit_posts] if limit_posts
+      if limit_posts
+        limit = self.posts.length < limit_posts ? self.posts.length : limit_posts
+        self.posts = self.posts[-limit, limit]
+      end
     end
 
     # Run each of the Generators.
@@ -189,12 +195,13 @@ module Jekyll
     #
     # Returns nothing.
     def render
+      payload = site_payload
       self.posts.each do |post|
-        post.render(self.layouts, site_payload)
+        post.render(self.layouts, payload)
       end
 
       self.pages.each do |page|
-        page.render(self.layouts, site_payload)
+        page.render(self.layouts, payload)
       end
 
       self.categories.values.map { |ps| ps.sort! { |a, b| b <=> a } }
@@ -250,7 +257,7 @@ module Jekyll
       end
     end
 
-    # Constructs a Hash of Posts indexed by the specified Post attribute.
+    # Construct a Hash of Posts indexed by the specified Post attribute.
     #
     # post_attr - The String name of the Post attribute.
     #
@@ -300,12 +307,12 @@ module Jekyll
     # or are excluded in the site configuration, unless they are web server
     # files such as '.htaccess'.
     #
-    # entries - The Array of file/directory entries to filter.
+    # entries - The Array of String file/directory entries to filter.
     #
     # Returns the Array of filtered entries.
     def filter_entries(entries)
       entries = entries.reject do |e|
-        unless ['.htaccess'].include?(e)
+        unless self.include.include?(e)
           ['.', '_', '#'].include?(e[0..0]) ||
           e[-1..-1] == '~' ||
           self.exclude.include?(e) ||