jekyll-import 0.12.0 → 0.13.0

data/lib/jekyll-import/importers/posterous.rb

@@ -1,56 +1,55 @@
 module JekyllImport
   module Importers
     class Posterous < Importer
-
       def self.specify_options(c)
-        c.option 'email', '--email EMAIL', 'Posterous email address'
-        c.option 'password', '--password PW', 'Posterous password'
-        c.option 'api_token', '--token TOKEN', 'Posterous API Token'
+        c.option "email", "--email EMAIL", "Posterous email address"
+        c.option "password", "--password PW", "Posterous password"
+        c.option "api_token", "--token TOKEN", "Posterous API Token"
       end
 
       def self.require_deps
-        JekyllImport.require_with_fallback(%w[
+        JekyllImport.require_with_fallback(%w(
           rubygems
           jekyll
           fileutils
           uri
           json
           net/http
-        ])
+        ))
       end
 
       def self.fetch(uri_str, limit = 10)
         # You should choose better exception.
-        raise ArgumentError, 'Stuck in a redirect loop. Please double check your email and password' if limit == 0
+        raise ArgumentError, "Stuck in a redirect loop. Please double check your email and password" if limit.zero?
 
         response = nil
-        Net::HTTP.start('posterous.com') do |http|
+        Net::HTTP.start("posterous.com") do |http|
           req = Net::HTTP::Get.new(uri_str)
           req.basic_auth @email, @pass
           response = http.request(req)
         end
 
         case response
-          when Net::HTTPSuccess     then response
-          when Net::HTTPRedirection then fetch(response['location'], limit - 1)
-          else response.error!
+        when Net::HTTPSuccess     then response
+        when Net::HTTPRedirection then fetch(response["location"], limit - 1)
+        else response.error!
         end
       end
 
       def self.fetch_images(directory, imgs)
         def self.fetch_one(url, limit = 10)
-          raise ArgumentError, 'HTTP redirect too deep' if limit == 0
+          raise ArgumentError, "HTTP redirect too deep" if limit.zero?
           response = Net::HTTP.get_response(URI.parse(url))
           case response
           when Net::HTTPSuccess     then response.body
-          when Net::HTTPRedirection then self.fetch_one(response['location'], limit - 1)
+          when Net::HTTPRedirection then self.fetch_one(response["location"], limit - 1)
           else
             response.error!
           end
         end
 
         FileUtils.mkdir_p directory
-        urls = Array.new
+        urls = []
         imgs.each do |img|
           fullurl = img["full"]["url"]
           uri = URI.parse(fullurl)
@@ -66,12 +65,14 @@ module JekyllImport
       end
 
       def self.process(options)
-        email     = options.fetch('email')
-        pass      = options.fetch('password')
-        api_token = options.fetch('api_token')
-
-        @email, @pass, @api_token = email, pass, api_token
-        defaults = { :include_imgs => false, :blog => 'primary', :base_path => '/' }
+        email     = options.fetch("email")
+        pass      = options.fetch("password")
+        api_token = options.fetch("api_token")
+
+        @email = email
+        @pass = pass
+        @api_token = api_token
+        defaults = { :include_imgs => false, :blog => "primary", :base_path => "/" }
         opts = defaults.merge(opts)
         FileUtils.mkdir_p "_posts"
 
@@ -81,18 +82,18 @@ module JekyllImport
         while posts.any?
           posts.each do |post|
             title = post["title"]
-            slug = title.gsub(/[^[:alnum:]]+/, '-').downcase
+            slug = title.gsub(%r![^[:alnum:]]+!, "-").downcase
             date = Date.parse(post["display_date"])
             content = post["body_html"]
             published = !post["is_private"]
-            basename = "%02d-%02d-%02d-%s" % [date.year, date.month, date.day, slug]
-            name = basename + '.html'
+            basename = format("%02d-%02d-%02d-%s", date.year, date.month, date.day, slug)
+            name = basename + ".html"
 
             # Images:
             if opts[:include_imgs]
               post_imgs = post["media"]["images"]
               if post_imgs.any?
-                img_dir = "imgs/%s" % basename
+                img_dir = format("imgs/%s", basename)
                 img_urls = self.fetch_images(img_dir, post_imgs)
 
                 img_urls.map! do |url|
@@ -101,17 +102,17 @@ module JekyllImport
                 imgcontent = "<ol>\n" + img_urls.join("\n") + "</ol>\n"
 
                 # filter out "posterous-content", replacing with imgs:
-                content = content.sub(/\<p\>\[\[posterous-content:[^\]]+\]\]\<\/p\>/, imgcontent)
+                content = content.sub(%r!\<p\>\[\[posterous-content:[^\]]+\]\]\<\/p\>!, imgcontent)
               end
             end
 
             # Get the relevant fields as a hash, delete empty fields and convert
             # to YAML for the header
             data = {
-               'layout' => 'post',
-               'title' => title.to_s,
-               'published' => published
-             }.delete_if { |k,v| v.nil? || v == ''}.to_yaml
+              "layout"    => "post",
+              "title"     => title.to_s,
+              "published" => published,
+            }.delete_if { |_k, v| v.nil? || v == "" }.to_yaml
 
             # Write out the data and content to file
             File.open("_posts/#{name}", "w") do |f|

data/lib/jekyll-import/importers/rss.rb

@@ -2,24 +2,25 @@ module JekyllImport
   module Importers
     class RSS < Importer
       def self.specify_options(c)
-        c.option 'source', '--source NAME', 'The RSS file or URL to import'
+        c.option "source", "--source NAME", "The RSS file or URL to import"
+        c.option "tag", "--tag NAME", "Add a tag to posts"
       end
 
       def self.validate(options)
-        if options['source'].nil?
+        if options["source"].nil?
           abort "Missing mandatory option --source."
         end
       end
 
       def self.require_deps
-        JekyllImport.require_with_fallback(%w[
+        JekyllImport.require_with_fallback(%w(
           rss
           rss/1.0
           rss/2.0
           open-uri
           fileutils
           safe_yaml
-        ])
+        ))
       end
 
       # Process the import.
@@ -28,7 +29,7 @@ module JekyllImport
       #
       # Returns nothing.
       def self.process(options)
-        source = options.fetch('source')
+        source = options.fetch("source")
 
         content = ""
         open(source) { |s| content = s.read }
@@ -37,17 +38,19 @@ module JekyllImport
         raise "There doesn't appear to be any RSS items at the source (#{source}) provided." unless rss
 
         rss.items.each do |item|
-          formatted_date = item.date.strftime('%Y-%m-%d')
+          formatted_date = item.date.strftime("%Y-%m-%d")
           post_name = item.title.split(%r{ |!|/|:|&|-|$|,}).map do |i|
-            i.downcase if i != ''
-          end.compact.join('-')
+            i.downcase if i != ""
+          end.compact.join("-")
           name = "#{formatted_date}-#{post_name}"
 
           header = {
-            'layout' => 'post',
-            'title' => item.title
+            "layout" => "post",
+            "title"  => item.title,
           }
 
+          header["tag"] = options["tag"] if !options.to_s.empty?
+
           FileUtils.mkdir_p("_posts")
 
           File.open("_posts/#{name}.html", "w") do |f|

data/lib/jekyll-import/importers/s9y.rb

@@ -2,50 +2,49 @@ module JekyllImport
   module Importers
     class S9Y < Importer
       def self.specify_options(c)
-        c.option 'source', '--source SOURCE', 'The URL of the S9Y RSS feed'
+        c.option "source", "--source SOURCE", "The URL of the S9Y RSS feed"
       end
 
       def self.validate(options)
-        if options['source'].nil?
+        if options["source"].nil?
           abort "Missing mandatory option --source, e.g. --source \"http://blog.example.com/rss.php?version=2.0&all=1\""
         end
       end
 
       def self.require_deps
-        JekyllImport.require_with_fallback(%w[
+        JekyllImport.require_with_fallback(%w(
           open-uri
           rss
           fileutils
           safe_yaml
-        ])
+        ))
       end
 
       def self.process(options)
-        source = options.fetch('source')
+        source = options.fetch("source")
 
         FileUtils.mkdir_p("_posts")
 
-        text = ''
+        text = ""
         open(source) { |line| text = line.read }
         rss = ::RSS::Parser.parse(text)
 
         rss.items.each do |item|
-          post_url = item.link.match('.*(/archives/.*)')[1]
-          categories = item.categories.collect { |c| c.content }
+          post_url = item.link.match(".*(/archives/.*)")[1]
+          categories = item.categories.collect(&:content)
           content = item.content_encoded.strip
           date = item.date
           slug = item.link.match('.*/archives/[0-9]+-(.*)\.html')[1]
-          name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day,
-                                                 slug]
+          name = format("%02d-%02d-%02d-%s.markdown", date.year, date.month, date.day, slug)
 
           data = {
-            'layout' => 'post',
-            'title' => item.title,
-            'categories' => categories,
-            'permalink' => post_url,
-            's9y_link' => item.link,
-            'date' => item.date,
-          }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
+            "layout"     => "post",
+            "title"      => item.title,
+            "categories" => categories,
+            "permalink"  => post_url,
+            "s9y_link"   => item.link,
+            "date"       => item.date,
+          }.delete_if { |_k, v| v.nil? || v == "" }.to_yaml
 
           # Write out the data and content to file
           File.open("_posts/#{name}", "w") do |f|

data/lib/jekyll-import/importers/s9y_database.rb

@@ -1,32 +1,33 @@
 module JekyllImport
   module Importers
     class S9YDatabase < Importer
-
       def self.require_deps
         JekyllImport.require_with_fallback(
-          %w[
-          rubygems
+          %w(
+            rubygems
           sequel
           fileutils
           safe_yaml
           unidecode
-          ])
+          )
+        )
       end
 
       def self.specify_options(c)
-        c.option 'dbname', '--dbname DB', 'Database name (default: "")'
-        c.option 'socket', '--socket SOCKET', 'Database socket (default: "")'
-        c.option 'user', '--user USER', 'Database user name (default: "")'
-        c.option 'password', '--password PW', "Database user's password (default: "")"
-        c.option 'host', '--host HOST', 'Database host name (default: "localhost")'
-        c.option 'table_prefix', '--table_prefix PREFIX', 'Table prefix name (default: "serendipity_")'
-        c.option 'clean_entities', '--clean_entities', 'Whether to clean entities (default: true)'
-        c.option 'comments', '--comments', 'Whether to import comments (default: true)'
-        c.option 'categories', '--categories', 'Whether to import categories (default: true)'
-        c.option 'tags', '--tags', 'Whether to import tags (default: true)'
-        c.option 'drafts', '--drafts', 'Whether to export drafts as well'
-        c.option 'markdown', '--markdown', 'convert into markdown format (default: false)'
-        c.option 'permalinks', '--permalinks', 'preserve S9Y permalinks (default: false)'
+        c.option "dbname", "--dbname DB", "Database name (default: '')"
+        c.option "socket", "--socket SOCKET", "Database socket (default: '')"
+        c.option "user", "--user USER", "Database user name (default: '')"
+        c.option "password", "--password PW", "Database user's password (default: '')"
+        c.option "host", "--host HOST", "Database host name (default: 'localhost')"
+        c.option "port", "--port PORT", "Custom database port connect to (default: 3306)"
+        c.option "table_prefix", "--table_prefix PREFIX", "Table prefix name (default: 'serendipity_')"
+        c.option "clean_entities", "--clean_entities", "Whether to clean entities (default: true)"
+        c.option "comments", "--comments", "Whether to import comments (default: true)"
+        c.option "categories", "--categories", "Whether to import categories (default: true)"
+        c.option "tags", "--tags", "Whether to import tags (default: true)"
+        c.option "drafts", "--drafts", "Whether to export drafts as well"
+        c.option "markdown", "--markdown", "convert into markdown format (default: false)"
+        c.option "permalinks", "--permalinks", "preserve S9Y permalinks (default: false)"
       end
 
       # Main migrator function. Call this to perform the migration.
@@ -35,6 +36,7 @@ module JekyllImport
       # user::    The database user name
       # pass::    The database user's password
       # host::    The address of the MySQL database host. Default: 'localhost'
+      # port::    The port of the MySQL database server. Default: 3306
       # socket::  The database socket's path
       # options:: A hash table of configuration options.
       #
@@ -54,7 +56,7 @@ module JekyllImport
       # :tags::           If true, save the post's tags in its
       #                   YAML front matter. Default: true.
       # :extension::      Set the post extension. Default: "html"
-      # :drafts::  If true, export drafts as well
+      # :drafts::         If true, export drafts as well
       #                   Default: true.
       # :markdown::       If true, convert the content to markdown
       #                   Default: false
@@ -63,35 +65,42 @@ module JekyllImport
       #
       def self.process(opts)
         options = {
-          :user           => opts.fetch('user', ''),
-          :pass           => opts.fetch('password', ''),
-          :host           => opts.fetch('host', 'localhost'),
-          :socket         => opts.fetch('socket', nil),
-          :dbname         => opts.fetch('dbname', ''),
-          :table_prefix   => opts.fetch('table_prefix', 'serendipity_'),
-          :clean_entities => opts.fetch('clean_entities', true),
-          :comments       => opts.fetch('comments', true),
-          :categories     => opts.fetch('categories', true),
-          :tags           => opts.fetch('tags', true),
-          :extension      => opts.fetch('extension', 'html'),
-          :drafts         => opts.fetch('drafts', true),
-          :markdown       => opts.fetch('markdown', false),
-          :permalinks     => opts.fetch('permalinks', false),
+          :user           => opts.fetch("user", ""),
+          :pass           => opts.fetch("password", ""),
+          :host           => opts.fetch("host", "localhost"),
+          :port						=> opts.fetch("port", 3306),
+          :socket         => opts.fetch("socket", nil),
+          :dbname         => opts.fetch("dbname", ""),
+          :table_prefix   => opts.fetch("table_prefix", "serendipity_"),
+          :clean_entities => opts.fetch("clean_entities", true),
+          :comments       => opts.fetch("comments", true),
+          :categories     => opts.fetch("categories", true),
+          :tags           => opts.fetch("tags", true),
+          :extension      => opts.fetch("extension", "html"),
+          :drafts         => opts.fetch("drafts", true),
+          :markdown       => opts.fetch("markdown", false),
+          :permalinks     => opts.fetch("permalinks", false),
         }
 
         if options[:clean_entities]
-          options[:clean_entities] = require_if_available('htmlentities', 'clean_entities')
+          options[:clean_entities] = require_if_available("htmlentities", "clean_entities")
         end
 
         if options[:markdown]
-          options[:markdown] = require_if_available('reverse_markdown', 'markdown')
+          options[:markdown] = require_if_available("reverse_markdown", "markdown")
         end
 
         FileUtils.mkdir_p("_posts")
         FileUtils.mkdir_p("_drafts") if options[:drafts]
 
-        db = Sequel.mysql2(options[:dbname], :user => options[:user], :password => options[:pass],
-                           :socket => options[:socket], :host => options[:host], :encoding => 'utf8')
+        db = Sequel.mysql2(options[:dbname], 
+          :user     => options[:user], 
+          :password => options[:pass],
+          :socket   => options[:socket], 
+          :host     => options[:host], 
+          :port     => options[:port],
+          :encoding => "utf8"
+        )
 
         px = options[:table_prefix]
 
@@ -108,7 +117,7 @@ module JekyllImport
           page[:slug] = sluggify(page[:title])
 
           page_name_list[ page[:id] ] = {
-            :slug   => page[:slug]
+            :slug => page[:slug],
           }
         end
 
@@ -119,6 +128,7 @@ module JekyllImport
              entries.title          AS `title`,
              entries.timestamp      AS `timestamp`,
              entries.body           AS `body`,
+             entries.extended       AS `body_extended`,
              authors.realname     AS `author`,
              authors.username     AS `author_login`,
              authors.email        AS `author_email`
@@ -148,11 +158,14 @@ module JekyllImport
           slug = sluggify(title)
         end
 
-        status = post[:isdraft] == 'true' ? 'draft' : 'published'
+        status = post[:isdraft] == "true" ? "draft" : "published"
         date = Time.at(post[:timestamp]).utc || Time.now.utc
-        name = "%02d-%02d-%02d-%s.%s" % [date.year, date.month, date.day, slug, extension]
+        name = format("%02d-%02d-%02d-%s.%s", date.year, date.month, date.day, slug, extension)
 
         content = post[:body].to_s
+        unless post[:body_extended].to_s.empty?
+					content += "\n\n" + post[:body_extended].to_s
+        end
 
         if options[:clean_entities]
           content = clean_entities(content)
@@ -170,28 +183,28 @@ module JekyllImport
         # Get the relevant fields as a hash, delete empty fields and
         # convert to YAML for the header.
         data = {
-          'layout'        => post[:type].to_s,
-          'status'        => status.to_s,
-          'published'     => status.to_s == 'draft' ? nil : (status.to_s == 'published'),
-          'title'         => title.to_s,
-          'author'        => {
-            'display_name'=> post[:author].to_s,
-            'login'       => post[:author_login].to_s,
-            'email'       => post[:author_email].to_s
+          "layout"       => post[:type].to_s,
+          "status"       => status.to_s,
+          "published"    => status.to_s == "draft" ? nil : (status.to_s == "published"),
+          "title"        => title.to_s,
+          "author"       => {
+            "display_name" => post[:author].to_s,
+            "login"        => post[:author_login].to_s,
+            "email"        => post[:author_email].to_s,
           },
-          'author_login'  => post[:author_login].to_s,
-          'author_email'  => post[:author_email].to_s,
-          'date'          => date.to_s,
-          'permalink'     => options[:permalinks] ? permalink : nil,
-          'categories'    => options[:categories] ? categories : nil,
-          'tags'          => options[:tags] ? tags : nil,
-          'comments'      => options[:comments] ? comments : nil,
-        }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
-
-        if post[:type] == 'page'
+          "author_login" => post[:author_login].to_s,
+          "author_email" => post[:author_email].to_s,
+          "date"         => date.to_s,
+          "permalink"    => options[:permalinks] ? permalink : nil,
+          "categories"   => options[:categories] ? categories : nil,
+          "tags"         => options[:tags] ? tags : nil,
+          "comments"     => options[:comments] ? comments : nil,
+        }.delete_if { |_k, v| v.nil? || v == "" }.to_yaml
+
+        if post[:type] == "page"
           filename = page_path(post[:id], page_name_list) + "index.#{extension}"
           FileUtils.mkdir_p(File.dirname(filename))
-        elsif status == 'draft'
+        elsif status == "draft"
           filename = "_drafts/#{slug}.#{extension}"
         else
           filename = "_posts/#{name}"
@@ -206,13 +219,11 @@ module JekyllImport
       end
 
       def self.require_if_available(gem_name, option_name)
-        begin
-          require gem_name
-          return true
-        rescue LoadError
-          STDERR.puts "Could not require '#{gem_name}', so the :#{option_name} option is now disabled."
-          return true
-        end
+        require gem_name
+        return true
+      rescue LoadError
+        STDERR.puts "Could not require '#{gem_name}', so the :#{option_name} option is now disabled."
+        return true
       end
 
       def self.process_categories(db, options, post)
@@ -232,11 +243,11 @@ module JekyllImport
         )
 
         db[cquery].each_with_object([]) do |category, categories|
-          if options[:clean_entities]
-            categories << clean_entities(category[:name])
-          else
-            categories << category[:name]
-          end
+          categories << if options[:clean_entities]
+                          clean_entities(category[:name])
+                        else
+                          category[:name]
+                        end
         end
       end
 
@@ -273,14 +284,14 @@ module JekyllImport
           end
 
           comments << {
-            'id'           => comment[:id].to_i,
-            'author'       => comauthor,
-            'author_email' => comment[:author_email].to_s,
-            'author_url'   => comment[:author_url].to_s,
-            'date'         => comment[:date].to_s,
-            'content'      => comcontent,
+            "id"           => comment[:id].to_i,
+            "author"       => comauthor,
+            "author_email" => comment[:author_email].to_s,
+            "author_url"   => comment[:author_url].to_s,
+            "date"         => comment[:date].to_s,
+            "content"      => comcontent,
           }
-        end.sort!{ |a,b| a['id'] <=> b['id'] }
+        end.sort! { |a, b| a["id"] <=> b["id"] }
       end
 
       def self.process_tags(db, options, post)
@@ -298,11 +309,11 @@ module JekyllImport
         )
 
         db[cquery].each_with_object([]) do |tag, tags|
-          if options[:clean_entities]
-            tags << clean_entities(tag[:name])
-          else
-            tags << tag[:name]
-          end
+          tags << if options[:clean_entities]
+                    clean_entities(tag[:name])
+                  else
+                    tag[:name]
+                  end
         end
       end
 
@@ -326,7 +337,7 @@ module JekyllImport
         end
       end
 
-      def self.clean_entities( text )
+      def self.clean_entities(text)
         if text.respond_to?(:force_encoding)
           text.force_encoding("UTF-8")
         end
@@ -342,22 +353,20 @@ module JekyllImport
         text
       end
 
-      def self.sluggify( title )
-        title.to_ascii.downcase.gsub(/[^0-9A-Za-z]+/, " ").strip.gsub(" ", "-")
+      def self.sluggify(title)
+        title.to_ascii.downcase.gsub(%r![^0-9A-Za-z]+!, " ").strip.tr(" ", "-")
       end
 
-      def self.page_path( page_id, page_name_list )
+      def self.page_path(page_id, page_name_list)
         if page_name_list.key?(page_id)
           [
             page_name_list[page_id][:slug],
-            '/'
+            "/",
           ].join("")
         else
           ""
         end
       end
-
     end
   end
 end
-