jekyll-import 0.14.0 → 0.15.0

data/lib/jekyll-import/importers/roller.rb

@@ -0,0 +1,277 @@
+# frozen_string_literal: true
+
+module JekyllImport
+  module Importers
+    class Roller < Importer
+      def self.require_deps
+        JekyllImport.require_with_fallback(%w(
+          rubygems
+          sequel
+          fileutils
+          safe_yaml
+          unidecode
+        ))
+      end
+
+      def self.specify_options(c)
+        c.option "dbname",         "--dbname DB",      "Database name (default: '')"
+        c.option "socket",         "--socket SOCKET",  "Database socket (default: '')"
+        c.option "user",           "--user USER",      "Database user name (default: '')"
+        c.option "password",       "--password PW",    "Database user's password (default: '')"
+        c.option "host",           "--host HOST",      "Database host name (default: 'localhost')"
+        c.option "port",           "--port PORT",      "Database port number (default: '3306')"
+        c.option "clean_entities", "--clean_entities", "Whether to clean entities (default: true)"
+        c.option "comments",       "--comments",       "Whether to import comments (default: true)"
+        c.option "categories",     "--categories",     "Whether to import categories (default: true)"
+        c.option "tags",           "--tags",           "Whether to import tags (default: true)"
+
+        c.option "status",         "--status STATUS,STATUS2", Array,
+                 "Array of allowed statuses (default: ['PUBLISHED'], other options: 'DRAFT')"
+      end
+
+      # Main migrator function. Call this to perform the migration.
+      #
+      # dbname::  The name of the database
+      # user::    The database user name
+      # pass::    The database user's password
+      # host::    The address of the MySQL database host. Default: 'localhost'
+      # port::    The port number of the MySQL database. Default: '3306'
+      # socket::  The database socket's path
+      # options:: A hash table of configuration options.
+      #
+      # Supported options are:
+      #
+      # :clean_entities:: If true, convert non-ASCII characters to HTML
+      #                   entities in the posts, comments, titles, and
+      #                   names. Requires the 'htmlentities' gem to
+      #                   work. Default: true.
+      # :comments::       If true, migrate post comments too. Comments
+      #                   are saved in the post's YAML front matter.
+      #                   Default: true.
+      # :categories::     If true, save the post's categories in its
+      #                   YAML front matter. Default: true.
+      # :tags::           If true, save the post's tags in its
+      #                   YAML front matter. Default: true.
+      # :extension::      Set the post extension. Default: "html"
+      # :status::         Array of allowed post statuses. Only
+      #                   posts with matching status will be migrated.
+      #                   Known statuses are :PUBLISHED and :DRAFT
+      #                   If this is nil or an empty
+      #                   array, all posts are migrated regardless of
+      #                   status. Default: [:PUBLISHED].
+      #
+      def self.process(opts)
+        options = {
+          :user           => opts.fetch("user", ""),
+          :pass           => opts.fetch("password", ""),
+          :host           => opts.fetch("host", "localhost"),
+          :port           => opts.fetch("port", "3306"),
+          :socket         => opts.fetch("socket", nil),
+          :dbname         => opts.fetch("dbname", ""),
+          :clean_entities => opts.fetch("clean_entities", true),
+          :comments       => opts.fetch("comments", true),
+          :categories     => opts.fetch("categories", true),
+          :tags           => opts.fetch("tags", true),
+          :extension      => opts.fetch("extension", "html"),
+          :status         => opts.fetch("status", ["PUBLISHED"]).map(&:to_sym) # :DRAFT
+        }
+
+        if options[:clean_entities]
+          begin
+            require "htmlentities"
+          rescue LoadError
+            STDERR.puts "Could not require 'htmlentities', so the :clean_entities option is now disabled."
+            options[:clean_entities] = false
+          end
+        end
+
+        FileUtils.mkdir_p("_posts")
+        FileUtils.mkdir_p("_drafts") if options[:status].include? :DRAFT
+
+        db = Sequel.mysql2(options[:dbname],
+                           :user     => options[:user],
+                           :password => options[:pass],
+                           :socket   => options[:socket],
+                           :host     => options[:host],
+                           :port     => options[:port],
+                           :encoding => "utf8")
+
+        posts_query = "
+           SELECT
+             weblogentry.id           AS `id`,
+             weblogentry.status       AS `status`,
+             weblogentry.title        AS `title`,
+             weblogentry.anchor       AS `slug`,
+             weblogentry.updatetime   AS `date`,
+             weblogentry.text         AS `content`,
+             weblogentry.summary      AS `excerpt`,
+             weblogentry.categoryid   AS `categoryid`,
+             roller_user.fullname     AS `author`,
+             roller_user.username     AS `author_login`,
+             roller_user.emailaddress AS `author_email`,
+             weblog.handle            AS `site`
+           FROM weblogentry AS `weblogentry`
+             LEFT JOIN roller_user AS `roller_user`
+               ON weblogentry.creator = roller_user.username
+             LEFT JOIN weblog AS `weblog`
+               ON weblogentry.websiteid = weblog.id"
+
+        if options[:status] && !options[:status].empty?
+          status = options[:status][0]
+          posts_query += "
+           WHERE weblogentry.status = '#{status}'"
+          options[:status][1..-1].each do |stat|
+            posts_query += " OR
+             weblogentry.status = '#{stat}'"
+          end
+        end
+
+        db[posts_query].each do |post|
+          process_post(post, db, options)
+        end
+      end
+
+      def self.process_post(post, db, options)
+        extension = options[:extension]
+
+        title = post[:title]
+        title = clean_entities(title) if options[:clean_entities]
+
+        slug = post[:slug]
+        slug = sluggify(title) if !slug || slug.empty?
+
+        date = post[:date] || Time.now
+        name = format("%02d-%02d-%02d-%s.%s", date.year, date.month, date.day, slug, extension)
+
+        content = post[:content].to_s
+        content = clean_entities(content) if options[:clean_entities]
+
+        excerpt = post[:excerpt].to_s
+
+        permalink = "#{post[:site]}/entry/#{post[:slug]}"
+
+        categories = []
+        tags       = []
+
+        if options[:categories]
+          cquery =
+            "SELECT
+               weblogcategory.name AS `name`
+             FROM
+               weblogcategory AS `weblogcategory`
+             WHERE
+               weblogcategory.id = '#{post[:categoryid]}'"
+
+          db[cquery].each do |term|
+            categories << (options[:clean_entities] ? clean_entities(term[:name]) : term[:name])
+          end
+        end
+
+        if options[:tags]
+          cquery =
+            "SELECT
+               roller_weblogentrytag.name AS `name`
+             FROM
+               roller_weblogentrytag AS `roller_weblogentrytag`
+             WHERE
+               roller_weblogentrytag.entryid = '#{post[:id]}'"
+
+          db[cquery].each do |term|
+            tags << (options[:clean_entities] ? clean_entities(term[:name]) : term[:name])
+          end
+        end
+
+        comments = []
+
+        if options[:comments]
+          cquery =
+            "SELECT
+               id       AS `id`,
+               name     AS `author`,
+               email    AS `author_email`,
+               posttime AS `date`,
+               content  AS `content`
+             FROM roller_comment
+             WHERE
+               entryid = '#{post[:id]}' AND
+               status = 'APPROVED'"
+
+          db[cquery].each do |comment|
+            comcontent = comment[:content].to_s
+            comauthor  = comment[:author].to_s
+            comcontent.force_encoding("UTF-8") if comcontent.respond_to?(:force_encoding)
+
+            if options[:clean_entities]
+              comcontent = clean_entities(comcontent)
+              comauthor  = clean_entities(comauthor)
+            end
+
+            comments << {
+              "id"           => comment[:id].to_i,
+              "author"       => comauthor,
+              "author_email" => comment[:author_email].to_s,
+              "date"         => comment[:date].to_s,
+              "content"      => comcontent,
+            }
+          end
+
+          comments.sort! { |a, b| a["id"] <=> b["id"] }
+        end
+
+        # Get the relevant fields as a hash, delete empty fields and
+        # convert to YAML for the header.
+        data = {
+          "layout"       => post[:type].to_s,
+          "status"       => post[:status].to_s,
+          "published"    => post[:status].to_s == "DRAFT" ? nil : (post[:status].to_s == "PUBLISHED"),
+          "title"        => title.to_s,
+          "author"       => {
+            "display_name" => post[:author].to_s,
+            "login"        => post[:author_login].to_s,
+            "email"        => post[:author_email].to_s,
+          },
+          "author_login" => post[:author_login].to_s,
+          "author_email" => post[:author_email].to_s,
+          "excerpt"      => excerpt,
+          "id"           => post[:id],
+          "date"         => date.to_s,
+          "categories"   => options[:categories] ? categories : nil,
+          "tags"         => options[:tags] ? tags : nil,
+          "comments"     => options[:comments] ? comments : nil,
+          "permalink"    => permalink,
+        }.delete_if { |_k, v| v.nil? || v == "" }.to_yaml
+
+        filename = post[:status] == "DRAFT" ? "_drafts/#{slug}.md" : "_posts/#{name}"
+
+        # Write out the data and content to file
+        File.open(filename, "w") do |f|
+          f.puts data
+          f.puts "---"
+          f.puts Util.wpautop(content)
+        end
+      end
+
+      def self.clean_entities(text)
+        text.force_encoding("UTF-8") if text.respond_to?(:force_encoding)
+        text = HTMLEntities.new.encode(text, :named)
+        # We don't want to convert these, it would break all
+        # HTML tags in the post and comments.
+        text.gsub!("&amp;",  "&")
+        text.gsub!("&lt;",   "<")
+        text.gsub!("&gt;",   ">")
+        text.gsub!("&quot;", '"')
+        text.gsub!("&apos;", "'")
+        text.gsub!("&#47;",  "/")
+        text
+      end
+
+      def self.sluggify(title)
+        title.to_ascii.downcase.gsub(%r![^0-9a-z]+!, " ").strip.tr(" ", "-")
+      end
+
+      def self.page_path(_page_id)
+        ""
+      end
+    end
+  end
+end

data/lib/jekyll-import/importers/rss.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module JekyllImport
   module Importers
     class RSS < Importer
@@ -7,9 +9,7 @@ module JekyllImport
       end
 
       def self.validate(options)
-        if options["source"].nil?
-          abort "Missing mandatory option --source."
-        end
+        abort "Missing mandatory option --source." if options["source"].nil?
       end
 
       def self.require_deps
@@ -30,9 +30,11 @@ module JekyllImport
       # Returns nothing.
       def self.process(options)
         source = options.fetch("source")
+        frontmatter = options.fetch("frontmatter", [])
+        body = options.fetch("body", ["description"])
 
         content = ""
-        open(source) { |s| content = s.read }
+        URI.parse(source).open { |s| content = s.read }
         rss = ::RSS::Parser.parse(content, false)
 
         raise "There doesn't appear to be any RSS items at the source (#{source}) provided." unless rss
@@ -49,14 +51,24 @@ module JekyllImport
             "title"  => item.title,
           }
 
-          header["tag"] = options["tag"] if !options.to_s.empty?
+          header["tag"] = options["tag"] unless options.to_s.empty?
+
+          frontmatter.each do |value|
+            header[value] = item.send(value)
+          end
+
+          output = ""
+
+          body.each do |row|
+            output += item.send(row)
+          end
 
           FileUtils.mkdir_p("_posts")
 
           File.open("_posts/#{name}.html", "w") do |f|
             f.puts header.to_yaml
             f.puts "---\n\n"
-            f.puts item.description
+            f.puts output
           end
         end
       end

data/lib/jekyll-import/importers/s9y.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module JekyllImport
   module Importers
     class S9Y < Importer
@@ -26,7 +28,7 @@ module JekyllImport
         FileUtils.mkdir_p("_posts")
 
         text = ""
-        open(source) { |line| text = line.read }
+        URI.parse(source).open { |line| text = line.read }
         rss = ::RSS::Parser.parse(text)
 
         rss.items.each do |item|

data/lib/jekyll-import/importers/s9y_database.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module JekyllImport
   module Importers
     class S9YDatabase < Importer
@@ -5,29 +7,29 @@ module JekyllImport
         JekyllImport.require_with_fallback(
           %w(
             rubygems
-          sequel
-          fileutils
-          safe_yaml
-          unidecode
+            sequel
+            fileutils
+            safe_yaml
+            unidecode
           )
         )
       end
 
       def self.specify_options(c)
-        c.option "dbname", "--dbname DB", "Database name (default: '')"
-        c.option "socket", "--socket SOCKET", "Database socket (default: '')"
-        c.option "user", "--user USER", "Database user name (default: '')"
-        c.option "password", "--password PW", "Database user's password (default: '')"
-        c.option "host", "--host HOST", "Database host name (default: 'localhost')"
-        c.option "port", "--port PORT", "Custom database port connect to (default: 3306)"
-        c.option "table_prefix", "--table_prefix PREFIX", "Table prefix name (default: 'serendipity_')"
-        c.option "clean_entities", "--clean_entities", "Whether to clean entities (default: true)"
-        c.option "comments", "--comments", "Whether to import comments (default: true)"
-        c.option "categories", "--categories", "Whether to import categories (default: true)"
-        c.option "tags", "--tags", "Whether to import tags (default: true)"
-        c.option "drafts", "--drafts", "Whether to export drafts as well"
-        c.option "markdown", "--markdown", "convert into markdown format (default: false)"
-        c.option "permalinks", "--permalinks", "preserve S9Y permalinks (default: false)"
+        c.option "dbname",         "--dbname DB",           "Database name (default: '')"
+        c.option "socket",         "--socket SOCKET",       "Database socket (default: '')"
+        c.option "user",           "--user USER",           "Database user name (default: '')"
+        c.option "password",       "--password PW",         "Database user's password (default: '')"
+        c.option "host",           "--host HOST",           "Database host name (default: 'localhost')"
+        c.option "port",           "--port PORT",           "Custom database port connect to (default: 3306)"
+        c.option "table_prefix",   "--table_prefix PREFIX", "Table prefix name (default: 'serendipity_')"
+        c.option "clean_entities", "--clean_entities",      "Whether to clean entities (default: true)"
+        c.option "comments",       "--comments",            "Whether to import comments (default: true)"
+        c.option "categories",     "--categories",          "Whether to import categories (default: true)"
+        c.option "tags",           "--tags",                "Whether to import tags (default: true)"
+        c.option "drafts",         "--drafts",              "Whether to export drafts as well"
+        c.option "markdown",       "--markdown",            "convert into markdown format (default: false)"
+        c.option "permalinks",     "--permalinks",          "preserve S9Y permalinks (default: false)"
       end
 
       # Main migrator function. Call this to perform the migration.
@@ -68,7 +70,7 @@ module JekyllImport
           :user           => opts.fetch("user", ""),
           :pass           => opts.fetch("password", ""),
           :host           => opts.fetch("host", "localhost"),
-          :port						=> opts.fetch("port", 3306),
+          :port           => opts.fetch("port", 3306),
           :socket         => opts.fetch("socket", nil),
           :dbname         => opts.fetch("dbname", ""),
           :table_prefix   => opts.fetch("table_prefix", "serendipity_"),
@@ -93,14 +95,13 @@ module JekyllImport
         FileUtils.mkdir_p("_posts")
         FileUtils.mkdir_p("_drafts") if options[:drafts]
 
-        db = Sequel.mysql2(options[:dbname], 
-          :user     => options[:user], 
-          :password => options[:pass],
-          :socket   => options[:socket], 
-          :host     => options[:host], 
-          :port     => options[:port],
-          :encoding => "utf8"
-        )
+        db = Sequel.mysql2(options[:dbname],
+                           :user     => options[:user],
+                           :password => options[:pass],
+                           :socket   => options[:socket],
+                           :host     => options[:host],
+                           :port     => options[:port],
+                           :encoding => "utf8")
 
         px = options[:table_prefix]
 
@@ -136,9 +137,7 @@ module JekyllImport
              LEFT JOIN #{px}authors AS `authors`
                ON entries.authorid = authors.authorid"
 
-        unless options[:drafts]
-          posts_query << "WHERE posts.isdraft = 'false'"
-        end
+        posts_query << "WHERE posts.isdraft = 'false'" unless options[:drafts]
 
         db[posts_query].each do |post|
           process_post(post, db, options, page_name_list)
@@ -149,31 +148,21 @@ module JekyllImport
         extension = options[:extension]
 
         title = post[:title]
-        if options[:clean_entities]
-          title = clean_entities(title)
-        end
+        title = clean_entities(title) if options[:clean_entities]
 
         slug = post[:slug]
-        if !slug || slug.empty?
-          slug = sluggify(title)
-        end
+        slug = sluggify(title) if !slug || slug.empty?
 
         status = post[:isdraft] == "true" ? "draft" : "published"
         date = Time.at(post[:timestamp]).utc || Time.now.utc
         name = format("%02d-%02d-%02d-%s.%s", date.year, date.month, date.day, slug, extension)
 
         content = post[:body].to_s
-        unless post[:body_extended].to_s.empty?
-					content += "\n\n" + post[:body_extended].to_s
-        end
+        content += "\n\n" + post[:body_extended].to_s unless post[:body_extended].to_s.empty?
 
-        if options[:clean_entities]
-          content = clean_entities(content)
-        end
+        content = clean_entities(content) if options[:clean_entities]
 
-        if options[:markdown]
-          content = ReverseMarkdown.convert(content)
-        end
+        content = ReverseMarkdown.convert(content) if options[:markdown]
 
         categories = process_categories(db, options, post)
         comments = process_comments(db, options, post)
@@ -220,10 +209,10 @@ module JekyllImport
 
       def self.require_if_available(gem_name, option_name)
         require gem_name
-        return true
+        true
       rescue LoadError
         STDERR.puts "Could not require '#{gem_name}', so the :#{option_name} option is now disabled."
-        return true
+        true
       end
 
       def self.process_categories(db, options, post)
@@ -274,9 +263,7 @@ module JekyllImport
           comcontent = comment[:content].to_s
           comauthor = comment[:author].to_s
 
-          if comcontent.respond_to?(:force_encoding)
-            comcontent.force_encoding("UTF-8")
-          end
+          comcontent.force_encoding("UTF-8") if comcontent.respond_to?(:force_encoding)
 
           if options[:clean_entities]
             comcontent = clean_entities(comcontent)
@@ -338,9 +325,7 @@ module JekyllImport
       end
 
       def self.clean_entities(text)
-        if text.respond_to?(:force_encoding)
-          text.force_encoding("UTF-8")
-        end
+        text.force_encoding("UTF-8") if text.respond_to?(:force_encoding)
         text = HTMLEntities.new.encode(text, :named)
         # We don't want to convert these, it would break all
         # HTML tags in the post and comments.