jekyll-import 0.21.0 → 0.23.0

data/lib/jekyll-import/importers/mt.rb

@@ -32,17 +32,17 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "engine",        "--engine ENGINE",          "Database engine, (default: 'mysql', postgres also supported)"
-        c.option "dbname",        "--dbname DB",              "Database name"
-        c.option "user",          "--user USER",              "Database user name"
-        c.option "password",      "--password PW",            "Database user's password, (default: '')"
-        c.option "host",          "--host HOST",              'Database host name (default: "localhost")'
-        c.option "port",          "--port PORT",              "Custom database port connect to (optional)"
-        c.option "blog_id",       "--blog_id ID",             "Specify a single Movable Type blog ID to import (default: all blogs)"
-        c.option "categories",    "--categories",             "If true, save post's categories in its YAML front matter. (default: true)"
+        c.option "dbname",        "--dbname DB",              "Database name."
+        c.option "user",          "--user USER",              "Database user name."
+        c.option "engine",        "--engine ENGINE",          "Database engine ('mysql' or 'postgres'). (default: 'mysql')"
+        c.option "password",      "--password PW",            "Database user's password. (default: '')"
+        c.option "host",          "--host HOST",              "Database host name. (default: 'localhost')"
+        c.option "port",          "--port PORT",              "Custom database port connect to. (default: null)"
+        c.option "blog_id",       "--blog_id ID",             "Specify a single Movable Type blog ID to import. (default: null (all blogs))"
+        c.option "categories",    "--categories",             "When true, save post's categories in its YAML front matter. (default: true)"
         c.option "src_encoding",  "--src_encoding ENCODING",  "Encoding of strings from database. (default: UTF-8)"
         c.option "dest_encoding", "--dest_encoding ENCODING", "Encoding of output strings. (default: UTF-8)"
-        c.option "comments",      "--comments",               "If true, output comments in _comments directory (default: false)"
+        c.option "comments",      "--comments",               "When true, output comments in `_comments` directory. (default: false)"
       end
 
       # By default this migrator will include posts for all your MovableType blogs.
@@ -242,7 +242,7 @@ module JekyllImport
           Sequel.sqlite(dbname)
         when "mysql", "postgres"
           db_connect_opts = {
-            :host     => options.fetch("host", "localhost"),
+            :host     => options.fetch("host", "127.0.0.1"),
             :user     => options.fetch("user"),
             :password => options.fetch("password", ""),
           }

data/lib/jekyll-import/importers/pebble.rb

@@ -0,0 +1,97 @@
+module JekyllImport
+  module Importers
+    class Pebble < Importer
+      def self.require_deps
+        JekyllImport.require_with_fallback(%w(
+          nokogiri
+          safe_yaml
+        ))
+      end
+
+      def self.specify_options(c) 
+        c.option "directory", "--directory PATH", "Pebble source directory"
+      end
+
+      def self.process(opts)
+        options = {
+          directory: opts.fetch("directory", "")
+        }
+
+        FileUtils.mkdir_p("_posts")
+        FileUtils.mkdir_p("_drafts")
+
+        traverse_posts_within(options[:directory]) do |file|
+          next if file.end_with?('categories.xml')
+          process_file(file)
+        end
+      end
+
+      def self.traverse_posts_within(directory, &block) 
+        Dir.foreach(directory) do |fd|
+          path = File.join(directory, fd)
+          if fd == '.' || fd == '..'
+            next
+          elsif File.directory?(path) 
+            traverse_posts_within(path, &block)
+          elsif path.end_with?('xml')
+            yield(path) if block_given?
+          else
+          end
+        end
+      end
+
+      def self.process_file(file)
+        xml = File.open(file) { |f| Nokogiri::XML(f) }
+        raise "There doesn't appear to be any XML items at the source (#{file}) provided." unless xml
+
+        doc = xml.xpath("blogEntry")
+
+        title = kebabize(doc.xpath('title').text).gsub('_', '-')
+        date = Date.parse(doc.xpath('date').text)
+
+        directory = "_posts"
+        name = "#{date.strftime('%Y-%m-%d')}-#{title}"
+
+        header = {
+          "layout" => 'post',
+          "title"  => doc.xpath("title").text,
+          "tags"   => doc.xpath("tags").text.split(", "),
+          "categories" => doc.xpath('category').text.split(', ')
+        }
+        header["render_with_liquid"] = false
+
+        path = File.join(directory, "#{name}.html")
+        File.open(path, "w") do |f|
+          f.puts header.to_yaml
+          f.puts "---\n\n"
+          f.puts doc.xpath("body").text
+        end
+
+        Jekyll.logger.info "Wrote file #{path} successfully!"
+      end
+
+      def self.kebabize(string)
+        kebab = '-'.freeze
+        string.gsub!(/[^\w\-_]+/, kebab)
+
+        unless kebab.nil? || kebab.empty?
+          if kebab == "-".freeze
+            re_duplicate_kebab        = /-{2,}/
+            re_leading_trailing_kebab = /^-|-$/
+          else
+            re_sep = Regexp.escape(kebab)
+            re_duplicate_kebab        = /#{re_sep}{2,}/
+            re_leading_trailing_kebab = /^#{re_sep}|#{re_sep}$/
+          end
+          # No more than one of the kebab in a row.
+          string.gsub!(re_duplicate_kebab, kebab)
+          # Remove leading/trailing kebab.
+          string.gsub!(re_leading_trailing_kebab, "".freeze)
+        end
+
+        string.downcase!
+        string
+      end
+    end
+  end
+end

data/lib/jekyll-import/importers/pluxml.rb

@@ -12,9 +12,9 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "source", "--source NAME", "The PluXML data directory to import"
-        c.option "layout", "--layout NAME", "The layout to apply"
-        c.option "avoid_liquid", "--avoid_liquid true", "Will add render_with_liquid: false in frontmatter"
+        c.option "source",       "--source NAME",  "The PluXml data directory to import."
+        c.option "layout",       "--layout NAME",  "The layout to apply. (default: 'post')"
+        c.option "avoid_liquid", "--avoid_liquid", "Will add `render_with_liquid: false` in front matter. (default: false)"
       end
 
       def self.validate(options)

data/lib/jekyll-import/importers/roller.rb

@@ -14,19 +14,19 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "dbname",         "--dbname DB",      "Database name (default: '')"
-        c.option "socket",         "--socket SOCKET",  "Database socket (default: '')"
-        c.option "user",           "--user USER",      "Database user name (default: '')"
-        c.option "password",       "--password PW",    "Database user's password (default: '')"
-        c.option "host",           "--host HOST",      "Database host name (default: 'localhost')"
-        c.option "port",           "--port PORT",      "Database port number (default: '3306')"
-        c.option "clean_entities", "--clean_entities", "Whether to clean entities (default: true)"
-        c.option "comments",       "--comments",       "Whether to import comments (default: true)"
-        c.option "categories",     "--categories",     "Whether to import categories (default: true)"
-        c.option "tags",           "--tags",           "Whether to import tags (default: true)"
+        c.option "dbname",         "--dbname DB",      "Database name."
+        c.option "user",           "--user USER",      "Database user name."
+        c.option "password",       "--password PW",    "Database user's password."
+        c.option "socket",         "--socket SOCKET",  "Database socket. (default: null)"
+        c.option "host",           "--host HOST",      "Database host name. (default: 'localhost')"
+        c.option "port",           "--port PORT",      "Database port number. (default: '3306')"
+        c.option "clean_entities", "--clean_entities", "Whether to clean entities. (default: true)"
+        c.option "comments",       "--comments",       "Whether to import comments. (default: true)"
+        c.option "categories",     "--categories",     "Whether to import categories. (default: true)"
+        c.option "tags",           "--tags",           "Whether to import tags. (default: true)"
 
         c.option "status",         "--status STATUS,STATUS2", Array,
-                 "Array of allowed statuses (default: ['PUBLISHED'], other options: 'DRAFT')"
+                 "Array of allowed statuses (either ['PUBLISHED'] or ['DRAFT']). (default: ['PUBLISHED'])"
       end
 
       # Main migrator function. Call this to perform the migration.
@@ -64,7 +64,7 @@ module JekyllImport
         options = {
           :user           => opts.fetch("user", ""),
           :pass           => opts.fetch("password", ""),
-          :host           => opts.fetch("host", "localhost"),
+          :host           => opts.fetch("host", "127.0.0.1"),
           :port           => opts.fetch("port", "3306"),
           :socket         => opts.fetch("socket", nil),
           :dbname         => opts.fetch("dbname", ""),

data/lib/jekyll-import/importers/rss.rb

@@ -4,13 +4,16 @@ module JekyllImport
   module Importers
     class RSS < Importer
       def self.specify_options(c)
-        c.option "source", "--source NAME", "The RSS file or URL to import"
-        c.option "tag", "--tag NAME", "Add a tag to posts"
-        c.option "render_audio", "--render_audio", "Render <audio> element as necessary"
+        c.option "source",         "--source NAME",      "The RSS file or URL to import."
+        c.option "tag",            "--tag NAME",         "Add a specific tag to all posts."
+        c.option "extract_tags",   "--extract_tags KEY", "Copies tags from the given subfield on the RSS `<item>` to front matter. (default: null)"
+        c.option "render_audio",   "--render_audio",     "Render `<audio>` element in posts for the enclosure URLs. (default: false)"
+        c.option "canonical_link", "--canonical_link",   "Add original link as `canonical_url` to post front matter. (default: false)"
       end
 
       def self.validate(options)
         abort "Missing mandatory option --source." if options["source"].nil?
+        abort "Provide either --tag or --extract_tags option." if options["extract_tags"] && options["tag"]
       end
 
       def self.require_deps
@@ -33,7 +36,7 @@ module JekyllImport
         source = options.fetch("source")
 
         content = ""
-        open(source) { |s| content = s.read }
+        URI.open(source) { |s| content = s.read }
         rss = ::RSS::Parser.parse(content, false)
 
         raise "There doesn't appear to be any RSS items at the source (#{source}) provided." unless rss
@@ -52,13 +55,14 @@ module JekyllImport
         post_name = Jekyll::Utils.slugify(item.title, :mode => "latin")
         name = "#{formatted_date}-#{post_name}"
         audio = render_audio && item.enclosure.url
+        canonical_link = options.fetch("canonical_link", false)
 
         header = {
-          "layout" => "post",
-          "title"  => item.title,
-        }
-
-        header["tag"] = options["tag"] unless options["tag"].nil? || options["tag"].empty?
+          "layout"        => "post",
+          "title"         => item.title,
+          "canonical_url" => (canonical_link ? item.link : nil),
+          "tag"           => get_tags(item, options),
+        }.compact
 
         frontmatter.each do |value|
           header[value] = item.send(value)
@@ -91,6 +95,21 @@ module JekyllImport
           f.puts output
         end
       end
+
+      def self.get_tags(item, options)
+        explicit_tag = options["tag"]
+        return explicit_tag unless explicit_tag.nil? || explicit_tag.empty?
+
+        tags_reference = options["extract_tags"]
+        return unless tags_reference
+
+        tags_from_feed = item.instance_variable_get("@#{tags_reference}")
+        return unless tags_from_feed.is_a?(Array)
+
+        tags = tags_from_feed.map { |feed_tag| feed_tag.content.downcase }
+        tags.empty? ? nil : tags.tap(&:uniq!)
+      end
+      private_class_method :get_tags
     end
   end
 end

data/lib/jekyll-import/importers/s9y_database.rb

@@ -17,25 +17,25 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "dbname",            "--dbname DB",           "Database name (default: '')"
-        c.option "socket",            "--socket SOCKET",       "Database socket (default: '')"
-        c.option "user",              "--user USER",           "Database user name (default: '')"
-        c.option "password",          "--password PW",         "Database user's password (default: '')"
-        c.option "host",              "--host HOST",           "Database host name (default: 'localhost')"
-        c.option "port",              "--port PORT",           "Custom database port connect to (default: 3306)"
-        c.option "table_prefix",      "--table_prefix PREFIX", "Table prefix name (default: 'serendipity_')"
-        c.option "clean_entities",    "--clean_entities",      "Whether to clean entities (default: true)"
-        c.option "comments",          "--comments",            "Whether to import comments (default: true)"
-        c.option "categories",        "--categories",          "Whether to import categories (default: true)"
-        c.option "tags",              "--tags",                "Whether to import tags (default: true)"
-        c.option "drafts",            "--drafts",              "Whether to export drafts as well"
-        c.option "markdown",          "--markdown",            "convert into markdown format (default: false)"
-        c.option "permalinks",        "--permalinks",          "preserve S9Y permalinks (default: false)"
-        c.option "excerpt_separator", "--excerpt_separator",   "Demarkation for excerpts (default: '<a id=\"extended\"></a>')"
-        c.option "includeentry",      "--includeentry",        "Replace macros from the includeentry plugin (default: false)"
-        c.option "imgfig",            "--imgfig",              "Replace nested img and youtube divs with HTML figure tags (default: true)"
-        c.option "linebreak",         "--linebreak",           "Line break processing: wp, nokogiri, ignore (default: wp)"
-        c.option "relative",          "--relative",            "Convert links with this prefix to relative (default:nil)"
+        c.option "dbname",            "--dbname DB",           "Database name. (default: '')"
+        c.option "socket",            "--socket SOCKET",       "Database socket. (default: '')"
+        c.option "user",              "--user USER",           "Database user name. (default: '')"
+        c.option "password",          "--password PW",         "Database user's password. (default: '')"
+        c.option "host",              "--host HOST",           "Database host name. (default: 'localhost')"
+        c.option "port",              "--port PORT",           "Custom database port connect to. (default: 3306)"
+        c.option "table_prefix",      "--table_prefix PREFIX", "Table prefix name. (default: 'serendipity_')"
+        c.option "clean_entities",    "--clean_entities",      "Whether to clean entities. (default: true)"
+        c.option "comments",          "--comments",            "Whether to import comments. (default: true)"
+        c.option "categories",        "--categories",          "Whether to import categories. (default: true)"
+        c.option "tags",              "--tags",                "Whether to import tags. (default: true)"
+        c.option "drafts",            "--drafts",              "Whether to export drafts as well. (default: true)"
+        c.option "markdown",          "--markdown",            "convert into markdown format. (default: false)"
+        c.option "permalinks",        "--permalinks",          "preserve S9Y permalinks. (default: false)"
+        c.option "excerpt_separator", "--excerpt_separator",   "Demarkation for excerpts. (default: '<a id=\"extended\"></a>')"
+        c.option "includeentry",      "--includeentry",        "Replace macros from the includeentry plugin. (default: false)"
+        c.option "imgfig",            "--imgfig",              "Replace nested img and youtube divs with HTML figure tags. (default: true)"
+        c.option "linebreak",         "--linebreak",           "Line break processing: wp, nokogiri, ignore. (default: wp)"
+        c.option "relative",          "--relative",            "Convert links with this prefix to relative. (default: nil)"
       end
 
       # Main migrator function. Call this to perform the migration.
@@ -99,7 +99,7 @@ module JekyllImport
         options = {
           :user              => opts.fetch("user", ""),
           :pass              => opts.fetch("password", ""),
-          :host              => opts.fetch("host", "localhost"),
+          :host              => opts.fetch("host", "127.0.0.1"),
           :port              => opts.fetch("port", 3306),
           :socket            => opts.fetch("socket", nil),
           :dbname            => opts.fetch("dbname", ""),

data/lib/jekyll-import/importers/textpattern.rb

@@ -27,17 +27,17 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "dbname",   "--dbname DB",   "Database name"
-        c.option "user",     "--user USER",   "Database user name"
-        c.option "password", "--password PW", "Database user's password"
-        c.option "host",     "--host HOST",   'Database host name (default: "localhost")'
+        c.option "dbname",   "--dbname DB",   "Database name."
+        c.option "user",     "--user USER",   "Database user name."
+        c.option "password", "--password PW", "Database user's password. (default: '')"
+        c.option "host",     "--host HOST",   "Database host name. (default: 'localhost')"
       end
 
       def self.process(options)
         dbname = options.fetch("dbname")
         user   = options.fetch("user")
         pass   = options.fetch("password", "")
-        host   = options.fetch("host", "localhost")
+        host   = options.fetch("host", "127.0.0.1")
 
         db = Sequel.mysql2(dbname, :user => user, :password => pass, :host => host, :encoding => "utf8")
 

data/lib/jekyll-import/importers/tumblr.rb

@@ -19,11 +19,11 @@ module JekyllImport
         end
 
         def specify_options(c)
-          c.option "url",            "--url URL",        "Tumblr URL"
-          c.option "format",         "--format FORMAT",  'Output format (default: "html")'
-          c.option "grab_images",    "--grab_images",    "Whether to grab images (default: false)"
-          c.option "add_highlights", "--add_highlights", "Whether to add highlights (default: false)"
-          c.option "rewrite_urls",   "--rewrite_urls",   "Whether to rewrite URLs (default: false)"
+          c.option "url",            "--url URL",        "Tumblr URL."
+          c.option "format",         "--format FORMAT",  "Output format. (default: 'html')"
+          c.option "grab_images",    "--grab_images",    "Whether to grab images. (default: false)"
+          c.option "add_highlights", "--add_highlights", "Whether to add highlights. (default: false)"
+          c.option "rewrite_urls",   "--rewrite_urls",   "Whether to rewrite URLs. (default: false)"
         end
 
         def process(options)

data/lib/jekyll-import/importers/typo.rb

@@ -31,11 +31,11 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "server",   "--server TYPE", 'Server type ("mysql" or "postgres")'
-        c.option "dbname",   "--dbname DB",   "Database name"
-        c.option "user",     "--user USER",   "Database user name"
-        c.option "password", "--password PW", "Database user's password (default: '')"
-        c.option "host",     "--host HOST",   "Database host name"
+        c.option "server",   "--server TYPE", "Server type ('mysql' or 'postgres')."
+        c.option "dbname",   "--dbname DB",   "Database name."
+        c.option "user",     "--user USER",   "Database user name."
+        c.option "password", "--password PW", "Database user's password. (default: '')"
+        c.option "host",     "--host HOST",   "Database host name. (default: 'localhost')"
       end
 
       def self.process(options)
@@ -43,7 +43,7 @@ module JekyllImport
         dbname = options.fetch("dbname")
         user   = options.fetch("user")
         pass   = options.fetch("password", "")
-        host   = options.fetch("host", "localhost")
+        host   = options.fetch("host", "127.0.0.1")
 
         FileUtils.mkdir_p "_posts"
         case server.intern

data/lib/jekyll-import/importers/wordpress.rb

@@ -14,23 +14,23 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "dbname",         "--dbname DB",           "Database name (default: '')"
-        c.option "socket",         "--socket SOCKET",       "Database socket (default: '')"
-        c.option "user",           "--user USER",           "Database user name (default: '')"
-        c.option "password",       "--password PW",         "Database user's password (default: '')"
-        c.option "host",           "--host HOST",           "Database host name (default: 'localhost')"
-        c.option "port",           "--port PORT",           "Database port number (default: '')"
-        c.option "table_prefix",   "--table_prefix PREFIX", "Table prefix name (default: 'wp_')"
-        c.option "site_prefix",    "--site_prefix PREFIX",  "Site prefix name (default: '')"
-        c.option "clean_entities", "--clean_entities",      "Whether to clean entities (default: true)"
-        c.option "comments",       "--comments",            "Whether to import comments (default: true)"
-        c.option "categories",     "--categories",          "Whether to import categories (default: true)"
-        c.option "tags",           "--tags",                "Whether to import tags (default: true)"
-        c.option "more_excerpt",   "--more_excerpt",        "Whether to use more excerpt (default: true)"
-        c.option "more_anchor",    "--more_anchor",         "Whether to use more anchor (default: true)"
+        c.option "dbname",         "--dbname DB",           "Database name. (default: '')"
+        c.option "socket",         "--socket SOCKET",       "Database socket. (default: '')"
+        c.option "user",           "--user USER",           "Database user name. (default: '')"
+        c.option "password",       "--password PW",         "Database user's password. (default: '')"
+        c.option "host",           "--host HOST",           "Database host name. (default: 'localhost')"
+        c.option "port",           "--port PORT",           "Database port number. (default: '')"
+        c.option "table_prefix",   "--table_prefix PREFIX", "Table prefix name. (default: 'wp_')"
+        c.option "site_prefix",    "--site_prefix PREFIX",  "Site prefix name. (default: '')"
+        c.option "clean_entities", "--clean_entities",      "Whether to clean entities. (default: true)"
+        c.option "comments",       "--comments",            "Whether to import comments. (default: true)"
+        c.option "categories",     "--categories",          "Whether to import categories. (default: true)"
+        c.option "tags",           "--tags",                "Whether to import tags. (default: true)"
+        c.option "more_excerpt",   "--more_excerpt",        "Whether to use more excerpt. (default: true)"
+        c.option "more_anchor",    "--more_anchor",         "Whether to use more anchor. (default: true)"
 
         c.option "status",         "--status STATUS,STATUS2", Array,
-                 "Array of allowed statuses (default: ['publish'], other options: 'draft', 'private', 'revision')"
+                 "Array of allowed statuses ('publish', 'draft', 'private', 'revision'). (default: ['publish'])"
       end
 
       # Main migrator function. Call this to perform the migration.
@@ -81,7 +81,7 @@ module JekyllImport
         options = {
           :user           => opts.fetch("user", ""),
           :pass           => opts.fetch("password", ""),
-          :host           => opts.fetch("host", "localhost"),
+          :host           => opts.fetch("host", "127.0.0.1"),
           :port           => opts.fetch("port", "3306"),
           :socket         => opts.fetch("socket", nil),
           :dbname         => opts.fetch("dbname", ""),

data/lib/jekyll-import/importers/wordpressdotcom.rb

@@ -16,9 +16,9 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "source",          "--source FILE",          'WordPress export XML file (default: "wordpress.xml")'
-        c.option "no_fetch_images", "--no-fetch-images",      "Do not fetch the images referenced in the posts"
-        c.option "assets_folder",   "--assets_folder FOLDER", "Folder where assets such as images will be downloaded to (default: assets)"
+        c.option "source",          "--source FILE",          "WordPress export XML file (default: 'wordpress.xml')"
+        c.option "no_fetch_images", "--no-fetch-images",      "Do not fetch the images referenced in the posts (default: false)"
+        c.option "assets_folder",   "--assets_folder FOLDER", "Folder where assets such as images will be downloaded to (default: 'assets')"
       end
 
       # Will modify post DOM tree
@@ -28,7 +28,7 @@ module JekyllImport
 
         Jekyll.logger.info "Downloading images for ", title
         images.each do |i|
-          uri = i["src"]
+          uri = URI::DEFAULT_PARSER.escape(i["src"])
 
           dst = File.join(assets_folder, File.basename(uri))
           i["src"] = File.join("{{ site.baseurl }}", dst)

data/lib/jekyll-import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JekyllImport
-  VERSION = "0.21.0"
+  VERSION = "0.23.0"
 end