jekyll-import 0.20.0 → 0.22.0

data/lib/jekyll-import/importers/drupal_common.rb

@@ -16,16 +16,16 @@ module JekyllImport
         DEFAULTS = {
           "engine"   => "mysql",
           "password" => "",
-          "host"     => "localhost",
+          "host"     => "127.0.0.1",
           "prefix"   => "",
           "port"     => "3306",
           "types"    => %w(blog story article),
         }.freeze
 
         def specify_options(c)
-          c.option "engine",   "--engine [mysql|postgresql]", "Database engine (default: #{DEFAULTS["engine"].inspect})"
           c.option "dbname",   "--dbname DB",                 "Database name"
           c.option "user",     "--user USER",                 "Database user name"
+          c.option "engine",   "--engine [mysql|postgresql]", "Database engine (default: #{DEFAULTS["engine"].inspect})"
           c.option "password", "--password PW",               "Database user's password (default: #{DEFAULTS["password"].inspect})"
           c.option "host",     "--host HOST",                 "Database host name (default: #{DEFAULTS["host"].inspect})"
           c.option "port",     "--port PORT",                 "Database port name (default: #{DEFAULTS["port"].inspect})"
@@ -47,9 +47,9 @@ module JekyllImport
         end
 
         def process(options)
-          engine = options.fetch("engine", DEFAULTS["engine"])
           dbname = options.fetch("dbname")
           user   = options.fetch("user")
+          engine = options.fetch("engine",   DEFAULTS["engine"])
           pass   = options.fetch("password", DEFAULTS["password"])
           host   = options.fetch("host",     DEFAULTS["host"])
           port   = options.fetch("port",     DEFAULTS["port"])
@@ -68,6 +68,7 @@ module JekyllImport
           src_dir = conf["source"]
 
           dirs = {
+            :_aliases => src_dir,
             :_posts   => File.join(src_dir, "_posts").to_s,
             :_drafts  => File.join(src_dir, "_drafts").to_s,
             :_layouts => Jekyll.sanitized_path(src_dir, conf["layouts_dir"].to_s),
@@ -146,10 +147,10 @@ module JekyllImport
 
               if partition.first.length.positive?
                 dir = "#{partition.first}/"
-                FileUtils.mkdir_p partition.first
+                FileUtils.mkdir_p "#{dirs[:_aliases]}/#{dir}"
               end
 
-              File.open("#{dir}#{file}.md", "w") do |f|
+              File.open("#{dirs[:_aliases]}/#{dir}#{file}.md", "w") do |f|
                 f.puts "---"
                 f.puts "layout: refresh"
                 f.puts "permalink: #{dir}#{file}/"

data/lib/jekyll-import/importers/easyblog.rb

@@ -10,12 +10,12 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "dbname",   "--dbname",   "Database name"
-        c.option "user",     "--user",     "Database user name"
-        c.option "password", "--password", "Database user's password (default: '')"
-        c.option "host",     "--host",     "Database host name"
-        c.option "section",  "--section",  "Table prefix name"
-        c.option "prefix",   "--prefix",   "Table prefix name"
+        c.option "dbname",   "--dbname",   "Database name."
+        c.option "user",     "--user",     "Database user name."
+        c.option "password", "--password", "Database user's password. (default: '')"
+        c.option "host",     "--host",     "Database host name. (default: 'localhost')"
+        c.option "section",  "--section",  "Section ID. (default: '1')"
+        c.option "prefix",   "--prefix",   "Table prefix name. (default: 'jos_')"
       end
 
       def self.require_deps
@@ -32,7 +32,7 @@ module JekyllImport
         dbname  = options.fetch("dbname")
         user    = options.fetch("user")
         pass    = options.fetch("password", "")
-        host    = options.fetch("host", "localhost")
+        host    = options.fetch("host", "127.0.0.1")
         section = options.fetch("section", "1")
         table_prefix = options.fetch("prefix", "jos_")
 

data/lib/jekyll-import/importers/enki.rb

@@ -20,10 +20,10 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "dbname",   "--dbname",   "Database name"
-        c.option "user",     "--user",     "Database name"
-        c.option "password", "--password", 'Database name (default: "")'
-        c.option "host",     "--host",     "Database name"
+        c.option "dbname",   "--dbname",   "Database name."
+        c.option "user",     "--user",     "User name."
+        c.option "password", "--password", "Database password. (default: '')"
+        c.option "host",     "--host",     "Database host name. (default: 'localhost')"
       end
 
       def self.require_deps

data/lib/jekyll-import/importers/joomla.rb

@@ -10,13 +10,13 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "dbname",   "--dbname",   "Database name"
-        c.option "user",     "--user",     "Database user name"
-        c.option "password", "--password", "Database user's password (default: '')"
-        c.option "host",     "--host",     "Database host name"
-        c.option "port",     "--port",     "Database port"
-        c.option "section",  "--section",  "Table prefix name"
-        c.option "prefix",   "--prefix",   "Table prefix name"
+        c.option "dbname",   "--dbname",   "Database name."
+        c.option "user",     "--user",     "Database user name."
+        c.option "password", "--password", "Database user's password. (default: '')"
+        c.option "host",     "--host",     "Database host name. (default: 'localhost')"
+        c.option "port",     "--port",     "Database port. (default: '3306')"
+        c.option "section",  "--section",  "Section ID. (default: '1')"
+        c.option "prefix",   "--prefix",   "Table prefix name. (default: 'jos_')"
       end
 
       def self.require_deps
@@ -33,7 +33,7 @@ module JekyllImport
         dbname  = options.fetch("dbname")
         user    = options.fetch("user")
         pass    = options.fetch("password", "")
-        host    = options.fetch("host", "localhost")
+        host    = options.fetch("host", "127.0.0.1")
         port    = options.fetch("port", 3306).to_i
         section = options.fetch("section", "1")
         table_prefix = options.fetch("prefix", "jos_")

data/lib/jekyll-import/importers/joomla3.rb

@@ -10,13 +10,13 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "dbname",   "--dbname",   "Database name"
-        c.option "user",     "--user",     "Database user name"
-        c.option "password", "--password", "Database user's password (default: '')"
-        c.option "host",     "--host",     "Database host name"
-        c.option "port",     "--port",     "Database port"
-        c.option "category", "--category", "ID of the category"
-        c.option "prefix",   "--prefix",   "Table prefix name"
+        c.option "dbname",   "--dbname",   "Database name."
+        c.option "user",     "--user",     "Database user name."
+        c.option "password", "--password", "Database user's password. (default: '')"
+        c.option "host",     "--host",     "Database host name. (default: 'localhost')"
+        c.option "port",     "--port",     "Database port. (default: '3306')"
+        c.option "category", "--category", "ID of the category. (default: '0')"
+        c.option "prefix",   "--prefix",   "Table prefix name. (default: 'jos_')"
       end
 
       def self.require_deps
@@ -33,7 +33,7 @@ module JekyllImport
         dbname = options.fetch("dbname")
         user   = options.fetch("user")
         pass   = options.fetch("password", "")
-        host   = options.fetch("host", "localhost")
+        host   = options.fetch("host", "127.0.0.1")
         port   = options.fetch("port", 3306).to_i
         cid    = options.fetch("category", 0)
         table_prefix = options.fetch("prefix", "jos_")

data/lib/jekyll-import/importers/jrnl.rb

@@ -12,10 +12,10 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "file",        "--file FILENAME",      'Journal file (default: "~/journal.txt")'
-        c.option "time_format", "--time_format FORMAT", 'Time format of your journal (default: "%Y-%m-%d %H:%M")'
-        c.option "extension",   "--extension EXT",      'Output extension (default: "md")'
-        c.option "layout",      "--layout NAME",        'Output post layout (default: "post")'
+        c.option "file",        "--file FILENAME",      "Journal file. (default: '~/journal.txt')"
+        c.option "time_format", "--time_format FORMAT", "Time format of your journal. (default: '%Y-%m-%d %H:%M')"
+        c.option "extension",   "--extension EXT",      "Output extension. (default: 'md')"
+        c.option "layout",      "--layout NAME",        "Output post layout. (default: 'post')"
       end
 
       # Reads a jrnl file and creates a new post for each entry

data/lib/jekyll-import/importers/marley.rb

@@ -28,7 +28,7 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "marley_data_dir", "--marley_data_dir DIR", "The dir containing your marley data"
+        c.option "marley_data_dir", "--marley_data_dir DIR", "The dir containing your marley data."
       end
 
       def self.process(options)

data/lib/jekyll-import/importers/medium.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module JekyllImport
+  module Importers
+    class Medium < Importer
+      def self.specify_options(c)
+        c.option "username",       "--username NAME",  "Medium username"
+        c.option "canonical_link", "--canonical_link", "Copy original link as canonical_url to post (default: false)"
+        c.option "render_audio",   "--render_audio",   "Render <audio> element in posts for the enclosure URLs (default: false)"
+      end
+
+      def self.validate(options)
+        abort "Missing mandatory option --username." if options["username"].nil?
+      end
+
+      def self.require_deps
+        Importers::RSS.require_deps
+      end
+
+      # Medium posts and associated metadata are exported as an RSS Feed. Hence invoke our RSS Importer to create the
+      # Jekyll source directory.
+      #
+      # "Tags" attached to a Medium post are exported under the markup `<item><category>...</category></item>` in the
+      # export feed. Therefore, configure the RSS Importer to always look for tags in the `<category></category>` field
+      # of an RSS item.
+      def self.process(options)
+        Importers::RSS.process({
+          "source"         => "https://medium.com/feed/@#{options.fetch("username")}",
+          "render_audio"   => options.fetch("render_audio", false),
+          "canonical_link" => options.fetch("canonical_link", false),
+          "extract_tags"   => "category",
+        })
+      end
+    end
+  end
+end

data/lib/jekyll-import/importers/mephisto.rb

@@ -14,7 +14,7 @@ module JekyllImport
           COPY jekyll TO STDOUT WITH CSV HEADER;
           ROLLBACK;
         SQL
-        command = %(psql -h #{c[:host] || "localhost"} -c "#{sql.strip}" #{c[:database]} #{c[:username]} -o #{c[:filename] || "posts.csv"})
+        command = %(psql -h #{c[:host] || "127.0.0.1"} -c "#{sql.strip}" #{c[:database]} #{c[:username]} -o #{c[:filename] || "posts.csv"})
         Jekyll.logger.info "Executing:", command
         `#{command}`
         CSV.process
@@ -40,7 +40,7 @@ module JekyllImport
         c.option "dbname",   "--dbname DB",   "Database name"
         c.option "user",     "--user USER",   "Database user name"
         c.option "password", "--password PW", "Database user's password (default: '')"
-        c.option "host",     "--host HOST",   'Database host name (default: "localhost")'
+        c.option "host",     "--host HOST",   "Database host name (default: 'localhost')"
       end
 
       # This query will pull blog posts from all entries across all blogs. If
@@ -61,7 +61,7 @@ module JekyllImport
         dbname = options.fetch("dbname")
         user   = options.fetch("user")
         pass   = options.fetch("password", "")
-        host   = options.fetch("host", "localhost")
+        host   = options.fetch("host", "127.0.0.1")
 
         db = Sequel.mysql2(dbname, :user     => user,
                                    :password => pass,

data/lib/jekyll-import/importers/mt.rb

@@ -32,17 +32,17 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "engine",        "--engine ENGINE",          "Database engine, (default: 'mysql', postgres also supported)"
-        c.option "dbname",        "--dbname DB",              "Database name"
-        c.option "user",          "--user USER",              "Database user name"
-        c.option "password",      "--password PW",            "Database user's password, (default: '')"
-        c.option "host",          "--host HOST",              'Database host name (default: "localhost")'
-        c.option "port",          "--port PORT",              "Custom database port connect to (optional)"
-        c.option "blog_id",       "--blog_id ID",             "Specify a single Movable Type blog ID to import (default: all blogs)"
-        c.option "categories",    "--categories",             "If true, save post's categories in its YAML front matter. (default: true)"
+        c.option "dbname",        "--dbname DB",              "Database name."
+        c.option "user",          "--user USER",              "Database user name."
+        c.option "engine",        "--engine ENGINE",          "Database engine ('mysql' or 'postgres'). (default: 'mysql')"
+        c.option "password",      "--password PW",            "Database user's password. (default: '')"
+        c.option "host",          "--host HOST",              "Database host name. (default: 'localhost')"
+        c.option "port",          "--port PORT",              "Custom database port connect to. (default: null)"
+        c.option "blog_id",       "--blog_id ID",             "Specify a single Movable Type blog ID to import. (default: null (all blogs))"
+        c.option "categories",    "--categories",             "When true, save post's categories in its YAML front matter. (default: true)"
         c.option "src_encoding",  "--src_encoding ENCODING",  "Encoding of strings from database. (default: UTF-8)"
         c.option "dest_encoding", "--dest_encoding ENCODING", "Encoding of output strings. (default: UTF-8)"
-        c.option "comments",      "--comments",               "If true, output comments in _comments directory (default: false)"
+        c.option "comments",      "--comments",               "When true, output comments in `_comments` directory. (default: false)"
       end
 
       # By default this migrator will include posts for all your MovableType blogs.
@@ -242,7 +242,7 @@ module JekyllImport
           Sequel.sqlite(dbname)
         when "mysql", "postgres"
           db_connect_opts = {
-            :host     => options.fetch("host", "localhost"),
+            :host     => options.fetch("host", "127.0.0.1"),
             :user     => options.fetch("user"),
             :password => options.fetch("password", ""),
           }

data/lib/jekyll-import/importers/pluxml.rb

@@ -12,9 +12,9 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "source", "--source NAME", "The PluXML data directory to import"
-        c.option "layout", "--layout NAME", "The layout to apply"
-        c.option "avoid_liquid", "--avoid_liquid true", "Will add render_with_liquid: false in frontmatter"
+        c.option "source",       "--source NAME",  "The PluXml data directory to import."
+        c.option "layout",       "--layout NAME",  "The layout to apply. (default: 'post')"
+        c.option "avoid_liquid", "--avoid_liquid", "Will add `render_with_liquid: false` in front matter. (default: false)"
       end
 
       def self.validate(options)

data/lib/jekyll-import/importers/roller.rb

@@ -14,19 +14,19 @@ module JekyllImport
       end
 
       def self.specify_options(c)
-        c.option "dbname",         "--dbname DB",      "Database name (default: '')"
-        c.option "socket",         "--socket SOCKET",  "Database socket (default: '')"
-        c.option "user",           "--user USER",      "Database user name (default: '')"
-        c.option "password",       "--password PW",    "Database user's password (default: '')"
-        c.option "host",           "--host HOST",      "Database host name (default: 'localhost')"
-        c.option "port",           "--port PORT",      "Database port number (default: '3306')"
-        c.option "clean_entities", "--clean_entities", "Whether to clean entities (default: true)"
-        c.option "comments",       "--comments",       "Whether to import comments (default: true)"
-        c.option "categories",     "--categories",     "Whether to import categories (default: true)"
-        c.option "tags",           "--tags",           "Whether to import tags (default: true)"
+        c.option "dbname",         "--dbname DB",      "Database name."
+        c.option "user",           "--user USER",      "Database user name."
+        c.option "password",       "--password PW",    "Database user's password."
+        c.option "socket",         "--socket SOCKET",  "Database socket. (default: null)"
+        c.option "host",           "--host HOST",      "Database host name. (default: 'localhost')"
+        c.option "port",           "--port PORT",      "Database port number. (default: '3306')"
+        c.option "clean_entities", "--clean_entities", "Whether to clean entities. (default: true)"
+        c.option "comments",       "--comments",       "Whether to import comments. (default: true)"
+        c.option "categories",     "--categories",     "Whether to import categories. (default: true)"
+        c.option "tags",           "--tags",           "Whether to import tags. (default: true)"
 
         c.option "status",         "--status STATUS,STATUS2", Array,
-                 "Array of allowed statuses (default: ['PUBLISHED'], other options: 'DRAFT')"
+                 "Array of allowed statuses (either ['PUBLISHED'] or ['DRAFT']). (default: ['PUBLISHED'])"
       end
 
       # Main migrator function. Call this to perform the migration.
@@ -64,7 +64,7 @@ module JekyllImport
         options = {
           :user           => opts.fetch("user", ""),
           :pass           => opts.fetch("password", ""),
-          :host           => opts.fetch("host", "localhost"),
+          :host           => opts.fetch("host", "127.0.0.1"),
           :port           => opts.fetch("port", "3306"),
           :socket         => opts.fetch("socket", nil),
           :dbname         => opts.fetch("dbname", ""),

data/lib/jekyll-import/importers/rss.rb

@@ -4,12 +4,16 @@ module JekyllImport
   module Importers
     class RSS < Importer
       def self.specify_options(c)
-        c.option "source", "--source NAME", "The RSS file or URL to import"
-        c.option "tag", "--tag NAME", "Add a tag to posts"
+        c.option "source",         "--source NAME",      "The RSS file or URL to import."
+        c.option "tag",            "--tag NAME",         "Add a specific tag to all posts."
+        c.option "extract_tags",   "--extract_tags KEY", "Copies tags from the given subfield on the RSS `<item>` to front matter. (default: null)"
+        c.option "render_audio",   "--render_audio",     "Render `<audio>` element in posts for the enclosure URLs. (default: false)"
+        c.option "canonical_link", "--canonical_link",   "Add original link as `canonical_url` to post front matter. (default: false)"
       end
 
       def self.validate(options)
         abort "Missing mandatory option --source." if options["source"].nil?
+        abort "Provide either --tag or --extract_tags option." if options["extract_tags"] && options["tag"]
       end
 
       def self.require_deps
@@ -30,49 +34,82 @@ module JekyllImport
       # Returns nothing.
       def self.process(options)
         source = options.fetch("source")
-        frontmatter = options.fetch("frontmatter", [])
-        body = options.fetch("body", ["description"])
 
         content = ""
-        open(source) { |s| content = s.read }
+        URI.open(source) { |s| content = s.read }
         rss = ::RSS::Parser.parse(content, false)
 
         raise "There doesn't appear to be any RSS items at the source (#{source}) provided." unless rss
 
         rss.items.each do |item|
-          formatted_date = item.date.strftime("%Y-%m-%d")
-          post_name = Jekyll::Utils.slugify(item.title, :mode => "latin")
-          name = "#{formatted_date}-#{post_name}"
+          write_rss_item(item, options)
+        end
+      end
 
-          header = {
-            "layout" => "post",
-            "title"  => item.title,
-          }
+      def self.write_rss_item(item, options)
+        frontmatter = options.fetch("frontmatter", [])
+        body = options.fetch("body", ["description"])
+        render_audio = options.fetch("render_audio", false)
 
-          header["tag"] = options["tag"] unless options["tag"].nil? || options["tag"].empty?
+        formatted_date = item.date.strftime("%Y-%m-%d")
+        post_name = Jekyll::Utils.slugify(item.title, :mode => "latin")
+        name = "#{formatted_date}-#{post_name}"
+        audio = render_audio && item.enclosure.url
+        canonical_link = options.fetch("canonical_link", false)
 
-          frontmatter.each do |value|
-            header[value] = item.send(value)
-          end
+        header = {
+          "layout"        => "post",
+          "title"         => item.title,
+          "canonical_url" => (canonical_link ? item.link : nil),
+          "tag"           => get_tags(item, options),
+        }.compact
 
-          output = +""
+        frontmatter.each do |value|
+          header[value] = item.send(value)
+        end
 
-          body.each do |row|
-            output << item.send(row).to_s
-          end
+        output = +""
 
-          output.strip!
-          output = item.content_encoded if output.empty?
+        body.each do |row|
+          output << item.send(row).to_s
+        end
+
+        output.strip!
+        output = item.content_encoded if output.empty?
+
+        FileUtils.mkdir_p("_posts")
 
-          FileUtils.mkdir_p("_posts")
+        File.open("_posts/#{name}.html", "w") do |f|
+          f.puts header.to_yaml
+          f.puts "---\n\n"
 
-          File.open("_posts/#{name}.html", "w") do |f|
-            f.puts header.to_yaml
-            f.puts "---\n\n"
-            f.puts output
+          if audio
+            f.puts <<~HTML
+              <audio controls="">
+                <source src="#{audio}" type="audio/mpeg">
+                Your browser does not support the audio element.
+              </audio>
+            HTML
           end
+
+          f.puts output
         end
       end
+
+      def self.get_tags(item, options)
+        explicit_tag = options["tag"]
+        return explicit_tag unless explicit_tag.nil? || explicit_tag.empty?
+
+        tags_reference = options["extract_tags"]
+        return unless tags_reference
+
+        tags_from_feed = item.instance_variable_get("@#{tags_reference}")
+        return unless tags_from_feed.is_a?(Array)
+
+        tags = tags_from_feed.map { |feed_tag| feed_tag.content.downcase }
+        tags.empty? ? nil : tags.tap(&:uniq!)
+      end
+      private_class_method :get_tags
     end
   end
 end