jekyll 0.8.0 → 0.9.0

data/History.txt

@@ -1,3 +1,11 @@
+== 0.9.0 / 2010-12-15
+  * Minor Enhancements
+    * Use OptionParser's [no-] functionality for better boolean parsing.
+    * Add Drupal migrator (#245)
+    * Complain about YAML and Liquid errors (#249)
+    * Remove orphaned files during regeneration (#247)
+    * Add Marley migrator (#28)
+
 == 0.8.0 / 2010-11-22
   * Minor Enhancements
     * Add wordpress.com importer (#207)

data/bin/jekyll

@@ -23,16 +23,12 @@ options = {}
 opts = OptionParser.new do |opts|
   opts.banner = help
 
-  opts.on("--safe", "Safe mode (default unsafe)") do
-    options['safe'] = true
+  opts.on("--[no-]safe", "Safe mode (default unsafe)") do |safe|
+    options['safe'] = safe
   end
 
-  opts.on("--auto", "Auto-regenerate") do
-    options['auto'] = true
-  end
-
-  opts.on("--no-auto", "No auto-regenerate") do
-    options['auto'] = false
+  opts.on("--[no-]auto", "Auto-regenerate") do |auto|
+    options['auto'] = auto
   end
 
   opts.on("--server [PORT]", "Start web server (default port 4000)") do |port|
@@ -44,12 +40,12 @@ opts = OptionParser.new do |opts|
       options['baseurl'] = baseurl
   end
 
-  opts.on("--lsi", "Use LSI for better related posts") do
-    options['lsi'] = true
+  opts.on("--[no-]lsi", "Use LSI for better related posts") do |lsi|
+    options['lsi'] = lsi
   end
 
-  opts.on("--pygments", "Use pygments to highlight code") do
-    options['pygments'] = true
+  opts.on("--[no-]pygments", "Use pygments to highlight code") do |pygments|
+    options['pygments'] = pygments
   end
 
   opts.on("--rdiscount", "Use rdiscount gem for Markdown") do
@@ -64,12 +60,8 @@ opts = OptionParser.new do |opts|
     options['time'] = Time.parse(time)
   end
 
-  opts.on("--future", "Render future dated posts") do
-    options['future'] = true
-  end
-
-  opts.on("--no-future", "Do not render future dated posts") do
-    options['future'] = false
+  opts.on("--[no-]future", "Render future dated posts") do |future|
+    options['future'] = future
   end
 
   opts.on("--permalink [TYPE]", "Use 'date' (default) for YYYY/MM/DD") do |style|

data/jekyll.gemspec

@@ -4,8 +4,8 @@ Gem::Specification.new do |s|
   s.rubygems_version = '1.3.5'
 
   s.name              = 'jekyll'
-  s.version           = '0.8.0'
-  s.date              = '2010-11-22'
+  s.version           = '0.9.0'
+  s.date              = '2010-12-15'
   s.rubyforge_project = 'jekyll'
 
   s.summary     = "A simple, blog aware, static site generator."
@@ -68,6 +68,8 @@ Gem::Specification.new do |s|
     lib/jekyll/generators/pagination.rb
     lib/jekyll/layout.rb
     lib/jekyll/migrators/csv.rb
+    lib/jekyll/migrators/drupal.rb
+    lib/jekyll/migrators/marley.rb
     lib/jekyll/migrators/mephisto.rb
     lib/jekyll/migrators/mt.rb
     lib/jekyll/migrators/textpattern.rb

data/lib/jekyll.rb

@@ -45,7 +45,7 @@ require_all 'jekyll/generators'
 require_all 'jekyll/tags'
 
 module Jekyll
-  VERSION = '0.8.0'
+  VERSION = '0.9.0'
 
   # Default options. Overriden by values in _config.yml or command-line opts.
   # (Strings rather symbols used for compatability with YAML).

data/lib/jekyll/convertible.rb

@@ -26,7 +26,11 @@ module Jekyll
       if self.content =~ /^(---\s*\n.*?\n?)^(---\s*$\n?)/m
         self.content = self.content[($1.size + $2.size)..-1]
 
-        self.data = YAML.load($1)
+        begin
+          self.data = YAML.load($1)
+        rescue => e
+          puts "YAML Exception: #{e.message}"
+        end
       end
 
       self.data ||= {}
@@ -63,7 +67,13 @@ module Jekyll
       # render and transform content (this becomes the final content of the object)
       payload["pygments_prefix"] = converter.pygments_prefix
       payload["pygments_suffix"] = converter.pygments_suffix
-      self.content = Liquid::Template.parse(self.content).render(payload, info)
+      
+      begin
+        self.content = Liquid::Template.parse(self.content).render(payload, info)
+      rescue => e
+        puts "Liquid Exception: #{e.message} in #{self.data["layout"]}"
+      end
+      
       self.transform
 
       # output keeps track of what will finally be written
@@ -73,7 +83,12 @@ module Jekyll
       layout = layouts[self.data["layout"]]
       while layout
         payload = payload.deep_merge({"content" => self.output, "page" => layout.data})
-        self.output = Liquid::Template.parse(layout.content).render(payload, info)
+
+        begin
+          self.output = Liquid::Template.parse(layout.content).render(payload, info)
+        rescue => e
+          puts "Liquid Exception: #{e.message} in #{self.data["layout"]}"
+        end
 
         layout = layouts[layout.data["layout"]]
       end

data/lib/jekyll/migrators/drupal.rb

@@ -0,0 +1,86 @@
+require 'rubygems'
+require 'sequel'
+require 'fileutils'
+require 'yaml'
+
+# NOTE: This converter requires Sequel and the MySQL gems.
+# The MySQL gem can be difficult to install on OS X. Once you have MySQL
+# installed, running the following commands should work:
+# $ sudo gem install sequel
+# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config
+
+module Jekyll
+  module Drupal
+
+    # Reads a MySQL database via Sequel and creates a post file for each
+    # post in wp_posts that has post_status = 'publish'.
+    # This restriction is made because 'draft' posts are not guaranteed to
+    # have valid dates.
+    QUERY = "SELECT node.nid, node.title, node_revisions.body, node.created, node.status FROM node, node_revisions WHERE (node.type = 'blog' OR node.type = 'story') AND node.vid = node_revisions.vid"
+
+    def self.process(dbname, user, pass, host = 'localhost')
+      db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+
+      FileUtils.mkdir_p "_posts"
+      FileUtils.mkdir_p "_drafts"
+
+      # Create the refresh layout
+      # Change the refresh url if you customized your permalink config
+      File.open("_layouts/refresh.html", "w") do |f|
+        f.puts <<EOF
+<!DOCTYPE html>
+<html>
+<head>
+<meta http-equiv="content-type" content="text/html; charset=utf-8" />
+<meta http-equiv="refresh" content="0;url={{ page.refresh_to_post_id }}.html" />
+</head>
+</html>
+EOF
+      end
+
+      db[QUERY].each do |post|
+        # Get required fields and construct Jekyll compatible name
+        node_id = post[:nid]
+        title = post[:title]
+        content = post[:body]
+        created = post[:created]
+        time = Time.at(created)
+        is_published = post[:status] == 1
+        dir = is_published ? "_posts" : "_drafts"
+        slug = title.strip.downcase.gsub(/(&|&amp;)/, ' and ').gsub(/[\s\.\/\\]/, '-').gsub(/[^\w-]/, '').gsub(/[-_]{2,}/, '-').gsub(/^[-_]/, '').gsub(/[-_]$/, '')
+        name = time.strftime("%Y-%m-%d-") + slug + '.md'
+
+        # Get the relevant fields as a hash, delete empty fields and convert
+        # to YAML for the header
+        data = {
+           'layout' => 'post',
+           'title' => title.to_s,
+           'created' => created,
+         }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+
+        # Write out the data and content to file
+        File.open("#{dir}/#{name}", "w") do |f|
+          f.puts data
+          f.puts "---"
+          f.puts content
+        end
+
+        # Make a file to redirect from the old Drupal URL
+        if is_published
+          FileUtils.mkdir_p "node/#{node_id}"
+          File.open("node/#{node_id}/index.md", "w") do |f|
+            f.puts "---"
+            f.puts "layout: refresh"
+            f.puts "refresh_to_post_id: /#{time.strftime("%Y/%m/%d/") + slug}"
+            f.puts "---"
+          end
+        end
+      end
+
+      # TODO: Make dirs & files for nodes of type 'page'
+        # Make refresh pages for these as well
+
+      # TODO: Make refresh dirs & files according to entries in url_alias table
+    end
+  end
+end

data/lib/jekyll/migrators/marley.rb

@@ -0,0 +1,53 @@
+require 'yaml'
+require 'fileutils'
+
+module Jekyll
+  module Marley
+
+    def self.regexp
+      { :id    => /^\d{0,4}-{0,1}(.*)$/,
+        :title => /^#\s*(.*)\s+$/,
+        :title_with_date => /^#\s*(.*)\s+\(([0-9\/]+)\)$/,
+        :published_on => /.*\s+\(([0-9\/]+)\)$/,
+        :perex => /^([^\#\n]+\n)$/,
+        :meta  => /^\{\{\n(.*)\}\}\n$/mi # Multiline Regexp 
+      }
+    end
+
+    def self.process(marley_data_dir)
+      raise ArgumentError, "marley dir #{marley_data_dir} not found" unless File.directory?(marley_data_dir)
+
+      FileUtils.mkdir_p "_posts"
+
+      posts = 0
+      Dir["#{marley_data_dir}/**/*.txt"].each do |f|
+        next unless File.exists?(f)
+
+        #copied over from marley's app/lib/post.rb
+        file_content  = File.read(f)
+        meta_content  = file_content.slice!( self.regexp[:meta] )
+        body          = file_content.sub( self.regexp[:title], '').sub( self.regexp[:perex], '').strip
+   
+        title = file_content.scan( self.regexp[:title] ).first.to_s.strip
+        prerex = file_content.scan( self.regexp[:perex] ).first.to_s.strip
+        published_on = DateTime.parse( post[:published_on] ) rescue File.mtime( File.dirname(f) )
+        meta          = ( meta_content ) ? YAML::load( meta_content.scan( self.regexp[:meta]).to_s ) : {}
+        meta['title'] = title
+        meta['layout'] = 'post'
+
+        formatted_date = published_on.strftime('%Y-%m-%d')
+        post_name =  File.dirname(f).split(%r{/}).last.gsub(/\A\d+-/, '')
+
+        name = "#{formatted_date}-#{post_name}" 
+        File.open("_posts/#{name}.markdown", "w") do |f|
+          f.puts meta.to_yaml
+          f.puts "---\n"
+          f.puts "\n#{prerex}\n\n" if prerex
+          f.puts body
+        end
+        posts += 1
+      end
+      "Created #{posts} posts!"
+    end
+  end
+end

data/lib/jekyll/page.rb

@@ -93,24 +93,25 @@ module Jekyll
         "url"        => File.join(@dir, self.url),
         "content"    => self.content })
     end
+    
+    # Obtain destination path.
+    #   +dest+ is the String path to the destination dir
+    #
+    # Returns destination file path.
+    def destination(dest)
+      # The url needs to be unescaped in order to preserve the correct filename
+      path = File.join(dest, @dir, CGI.unescape(self.url))
+      path = File.join(path, "index.html") if self.url =~ /\/$/
+      path
+    end
 
     # Write the generated page file to the destination directory.
-    #   +dest_prefix+ is the String path to the destination dir
-    #   +dest_suffix+ is a suffix path to the destination dir
+    #   +dest+ is the String path to the destination dir
     #
     # Returns nothing
-    def write(dest_prefix, dest_suffix = nil)
-      dest = File.join(dest_prefix, @dir)
-      dest = File.join(dest, dest_suffix) if dest_suffix
-      FileUtils.mkdir_p(dest)
-
-      # The url needs to be unescaped in order to preserve the correct filename
-      path = File.join(dest, CGI.unescape(self.url))
-      if self.url =~ /\/$/
-        FileUtils.mkdir_p(path)
-        path = File.join(path, "index.html")
-      end
-
+    def write(dest)
+      path = destination(dest)
+      FileUtils.mkdir_p(File.dirname(path))
       File.open(path, 'w') do |f|
         f.write(self.output)
       end

data/lib/jekyll/post.rb

@@ -177,22 +177,25 @@ module Jekyll
 
       do_layout(payload, layouts)
     end
+    
+    # Obtain destination path.
+    #   +dest+ is the String path to the destination dir
+    #
+    # Returns destination file path.
+    def destination(dest)
+      # The url needs to be unescaped in order to preserve the correct filename
+      path = File.join(dest, CGI.unescape(self.url))
+      path = File.join(path, "index.html") if template[/\.html$/].nil?
+      path
+    end
 
     # Write the generated post file to the destination directory.
     #   +dest+ is the String path to the destination dir
     #
     # Returns nothing
     def write(dest)
-      FileUtils.mkdir_p(File.join(dest, dir))
-
-      # The url needs to be unescaped in order to preserve the correct filename
-      path = File.join(dest, CGI.unescape(self.url))
-
-      if template[/\.html$/].nil?
-        FileUtils.mkdir_p(path)
-        path = File.join(path, "index.html")
-      end
-
+      path = destination(dest)
+      FileUtils.mkdir_p(File.dirname(path))
       File.open(path, 'w') do |f|
         f.write(self.output)
       end

data/lib/jekyll/site.rb

@@ -79,6 +79,7 @@ module Jekyll
       self.read
       self.generate
       self.render
+      self.cleanup
       self.write
     end
 
@@ -151,6 +152,36 @@ module Jekyll
     rescue Errno::ENOENT => e
       # ignore missing layout dir
     end
+    
+    # Remove orphaned files and empty directories in destination
+    #
+    # Returns nothing
+    def cleanup
+      # all files and directories in destination, including hidden ones
+      dest_files = []
+      Dir.glob(File.join(self.dest, "**", "*"), File::FNM_DOTMATCH) do |file|
+        dest_files << file unless file =~ /\/\.{1,2}$/
+      end
+
+      # files to be written
+      files = []
+      self.posts.each do |post|
+        files << post.destination(self.dest)
+      end
+      self.pages.each do |page|
+        files << page.destination(self.dest)
+      end
+      self.static_files.each do |sf|
+        files << sf.destination(self.dest)
+      end
+      
+      # adding files' parent directories
+      files.each { |file| files << File.dirname(file) unless files.include? File.dirname(file) }
+      
+      obsolete_files = dest_files - files
+      
+      FileUtils.rm_rf(obsolete_files)
+    end
 
     # Write static files, pages and posts
     #

data/lib/jekyll/static_file.rb

@@ -52,12 +52,11 @@ module Jekyll
     # Returns false if the file was not modified since last time (no-op).
     def write(dest)
       dest_path = destination(dest)
-      dest_dir = File.join(dest, @dir)
 
       return false if File.exist? dest_path and !modified?
       @@mtimes[path] = mtime
 
-      FileUtils.mkdir_p(dest_dir)
+      FileUtils.mkdir_p(File.dirname(dest_path))
       FileUtils.cp(path, dest_path)
 
       true

data/test/test_site.rb

@@ -132,6 +132,39 @@ class TestSite < Test::Unit::TestCase
       assert_equal includes, @site.filter_entries(excludes + includes)
     end
     
+    context 'with orphaned files in destination' do
+      setup do
+        clear_dest
+        @site.process
+        # generate some orphaned files:
+        # hidden file
+        File.open(dest_dir('.htpasswd'), 'w')
+        # single file
+        File.open(dest_dir('obsolete.html'), 'w')
+        # single file in sub directory
+        FileUtils.mkdir(dest_dir('qux'))
+        File.open(dest_dir('qux/obsolete.html'), 'w')
+        # empty directory
+        FileUtils.mkdir(dest_dir('quux'))
+      end
+      
+      teardown do
+        FileUtils.rm_f(dest_dir('.htpasswd'))
+        FileUtils.rm_f(dest_dir('obsolete.html'))
+        FileUtils.rm_rf(dest_dir('qux'))
+        FileUtils.rm_f(dest_dir('quux'))
+      end
+      
+      should 'remove orphaned files in destination' do
+        @site.process
+        assert !File.exist?(dest_dir('.htpasswd'))
+        assert !File.exist?(dest_dir('obsolete.html'))
+        assert !File.exist?(dest_dir('qux'))
+        assert !File.exist?(dest_dir('quux'))
+      end
+
+    end
+    
     context 'with an invalid markdown processor in the configuration' do
       should 'not throw an error at initialization time' do
         bad_processor = 'not a processor name'

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: jekyll
 version: !ruby/object:Gem::Version 
-  hash: 63
+  hash: 59
   prerelease: false
   segments: 
   - 0
-  - 8
+  - 9
   - 0
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors: 
 - Tom Preston-Werner
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-22 00:00:00 -08:00
+date: 2010-12-15 00:00:00 -08:00
 default_executable: jekyll
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -219,6 +219,8 @@ files:
 - lib/jekyll/generators/pagination.rb
 - lib/jekyll/layout.rb
 - lib/jekyll/migrators/csv.rb
+- lib/jekyll/migrators/drupal.rb
+- lib/jekyll/migrators/marley.rb
 - lib/jekyll/migrators/mephisto.rb
 - lib/jekyll/migrators/mt.rb
 - lib/jekyll/migrators/textpattern.rb