jekyll-import 0.19.0 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e0c80e9a51361d575c8282c0dd6f813d24cf36423fcbe891a4dcda8c2410003
-  data.tar.gz: 8067c22cc55a955283967999590cecc4ea5f6fd212005b62919d811cb76bcc9b
+  metadata.gz: e0098ade7230382d4787c65ab3c17607680119727e1e9cdca587274d94e3e084
+  data.tar.gz: 8f1050dfa7170aa7b08462c0132ab959ffdcda31e8c469929ce123aecb64909a
 SHA512:
-  metadata.gz: 9ea2f084457d3192cece261e1e98511e4cf0219e62afb5d3c2b50f8c957e76fe82f85ae9b5c55beebf4c8b6c15f8acbd17da00dca245253af5ddc3a9a84888b7
-  data.tar.gz: f5abebbb51120ba3877075eaf82f4a02c07e3253011b75c996e3f1c9aa5ab9533e66775f5041702bda739513ad1e5fa600775c5e3b7e60d452422edc85507aec
+  metadata.gz: f32b5ac48f88293a4703c7ece13d4a9886c598b4491fa9e8b750b02f4dfab405291c095355885f823a9b23551bc69e73015487371daa9fb6a9db7ec4783b88f0
+  data.tar.gz: 0ba737a7d8ff767eb1bcce6bffa8af357d6c66793b36e682edfcce05245e21dd0a12d5a3d250118ae9b00dc40857db50884d2365e2419a86efce7d97e0ac52af

data/lib/jekyll/commands/import.rb

@@ -64,7 +64,7 @@ module Jekyll
 
         def abort_on_invalid_migrator(migrator)
           warn "Sorry, '#{migrator}' isn't a valid migrator. Valid choices:"
-          IMPORTERS.keys.each { |k| warn "* #{k}" }
+          IMPORTERS.each_key { |k| warn "* #{k}" }
           raise "'#{migrator}' is not a valid migrator."
         end
       end

data/lib/jekyll-import/importer.rb

@@ -12,7 +12,7 @@ module JekyllImport
 
     def self.stringify_keys(hash)
       the_hash = hash.clone
-      the_hash.keys.each do |key|
+      hash.each_key do |key|
         the_hash[(key.to_s rescue key) || key] = the_hash.delete(key)
       end
       the_hash

data/lib/jekyll-import/importers/blogger.rb

@@ -11,11 +11,8 @@ module JekyllImport
       end
 
       def self.validate(options)
-        if options["source"].nil?
-          raise "Missing mandatory option: --source"
-        elsif !File.exist?(options["source"])
-          raise Errno::ENOENT, "File not found: #{options["source"]}"
-        end
+        raise "Missing mandatory option: --source" if options["source"].nil?
+        raise Errno::ENOENT, "File not found: #{options["source"]}" unless File.exist?(options["source"])
       end
 
       def self.require_deps
@@ -42,7 +39,6 @@ module JekyllImport
         source = options.fetch("source")
 
         listener = BloggerAtomStreamListener.new
-
         listener.leave_blogger_info = !options.fetch("no-blogger-info", false)
         listener.comments = options.fetch("comments", false)
 
@@ -52,7 +48,6 @@ module JekyllImport
         end
 
         options["original-url-base"] = listener.original_url_base
-
         postprocess(options)
       end
 
@@ -63,32 +58,32 @@ module JekyllImport
       # Returns nothing.
       def self.postprocess(options)
         # Replace internal link URL
-        if options.fetch("replace-internal-link", false)
-          original_url_base = options.fetch("original-url-base", nil)
-          if original_url_base
-            orig_url_pattern = Regexp.new(" href=([\"\'])(?:#{Regexp.escape(original_url_base)})?/([0-9]{4})/([0-9]{2})/([^\"\']+\.html)\\1")
-
-            Dir.glob("_posts/*.*") do |filename|
-              body = nil
-              File.open(filename, "r") do |f|
-                f.flock(File::LOCK_SH)
-                body = f.read
-              end
+        return unless options.fetch("replace-internal-link", false)
 
-              body.gsub!(orig_url_pattern) do
-                # for post_url
-                quote = Regexp.last_match(1)
-                post_file = Dir.glob("_posts/#{Regexp.last_match(2)}-#{Regexp.last_match(3)}-*-#{Regexp.last_match(4).to_s.tr("/", "-")}").first
-                raise "Could not found: _posts/#{Regexp.last_match(2)}-#{Regexp.last_match(3)}-*-#{Regexp.last_match(4).to_s.tr("/", "-")}" if post_file.nil?
+        original_url_base = options.fetch("original-url-base", nil)
+        return unless original_url_base
 
-                " href=#{quote}{{ site.baseurl }}{% post_url #{File.basename(post_file, ".html")} %}#{quote}"
-              end
+        orig_url_pattern = Regexp.new(" href=([\"\'])(?:#{Regexp.escape(original_url_base)})?/([0-9]{4})/([0-9]{2})/([^\"\']+\.html)\\1")
 
-              File.open(filename, "w") do |f|
-                f.flock(File::LOCK_EX)
-                f << body
-              end
-            end
+        Dir.glob("_posts/*.*") do |filename|
+          body = nil
+          File.open(filename, "r") do |f|
+            f.flock(File::LOCK_SH)
+            body = f.read
+          end
+
+          body.gsub!(orig_url_pattern) do
+            # for post_url
+            quote = Regexp.last_match(1)
+            post_file = Dir.glob("_posts/#{Regexp.last_match(2)}-#{Regexp.last_match(3)}-*-#{Regexp.last_match(4).to_s.tr("/", "-")}").first
+            raise "Could not found: _posts/#{Regexp.last_match(2)}-#{Regexp.last_match(3)}-*-#{Regexp.last_match(4).to_s.tr("/", "-")}" if post_file.nil?
+
+            " href=#{quote}{{ site.baseurl }}{% post_url #{File.basename(post_file, ".html")} %}#{quote}"
+          end
+
+          File.open(filename, "w") do |f|
+            f.flock(File::LOCK_EX)
+            f << body
           end
         end
       end
@@ -118,9 +113,7 @@ module JekyllImport
 
             @in_entry_elem = { :meta => {}, :body => nil }
           when "title"
-            if @in_entry_elem
-              raise 'only <title type="text"></title> is supported' if attrs["type"] != "text"
-            end
+            raise 'only <title type="text"></title> is supported' if @in_entry_elem && attrs["type"] != "text"
           when "category"
             if @in_entry_elem
               if attrs["scheme"] == "http://www.blogger.com/atom/ns#"
@@ -150,25 +143,23 @@ module JekyllImport
         end
 
         def text(text)
-          if @in_entry_elem
-            case @tag_bread.last
-            when "id"
-              @in_entry_elem[:meta][:id] = text
-            when "published"
-              @in_entry_elem[:meta][:published] = text
-            when "updated"
-              @in_entry_elem[:meta][:updated] = text
-            when "title"
-              @in_entry_elem[:meta][:title] = text
-            when "content"
-              @in_entry_elem[:body] = text
-            when "name"
-              @in_entry_elem[:meta][:author] = text if @tag_bread[-2..-1] == %w(author name)
-            when "app:draft"
-              if @tag_bread[-2..-1] == %w(app:control app:draft)
-                @in_entry_elem[:meta][:draft] = true if text == "yes"
-              end
-            end
+          return unless @in_entry_elem
+
+          case @tag_bread.last
+          when "id"
+            @in_entry_elem[:meta][:id] = text
+          when "published"
+            @in_entry_elem[:meta][:published] = text
+          when "updated"
+            @in_entry_elem[:meta][:updated] = text
+          when "title"
+            @in_entry_elem[:meta][:title] = text
+          when "content"
+            @in_entry_elem[:body] = text
+          when "name"
+            @in_entry_elem[:meta][:author] = text if @tag_bread[-2..-1] == %w(author name)
+          when "app:draft"
+            @in_entry_elem[:meta][:draft] = true if @tag_bread[-2..-1] == %w(app:control app:draft) && text == "yes"
           end
         end
 
@@ -186,7 +177,7 @@ module JekyllImport
 
                 FileUtils.mkdir_p(target_dir)
 
-                file_name = URI.decode("#{post_data[:filename]}.html")
+                file_name = URI.decode_www_form_component("#{post_data[:filename]}.html")
                 File.open(File.join(target_dir, file_name), "w") do |f|
                   f.flock(File::LOCK_EX)
 
@@ -258,25 +249,22 @@ module JekyllImport
             body = @in_entry_elem[:body]
 
             # body escaping associated with liquid
-            body.gsub!(%r!{{!, '{{ "{{" }}') if body =~ %r!{{!
-            body.gsub!(%r!{%!, '{{ "{%" }}') if body =~ %r!{%!
+            body.gsub!(%r!{{!, '{{ "{{" }}') if %r!{{!.match?(body)
+            body.gsub!(%r!{%!, '{{ "{%" }}') if %r!{%!.match?(body)
 
             { :filename => filename, :header => header, :body => body }
           elsif @in_entry_elem[:meta][:kind] == "comment"
             timestamp = Time.parse(@in_entry_elem[:meta][:published]).strftime("%Y-%m-%d")
-            if @in_entry_elem[:meta][:original_url]
-              @comment_seq ||= 1
+            raise "Original URL is missing" unless @in_entry_elem[:meta][:original_url]
 
-              original_uri = URI.parse(@in_entry_elem[:meta][:original_url])
-              original_path = original_uri.path.to_s
-              filename = format("%s-%s-%s", timestamp, File.basename(original_path, File.extname(original_path)), @comment_seq)
+            @comment_seq ||= 1
 
-              @comment_seq += 1
+            original_uri  = URI.parse(@in_entry_elem[:meta][:original_url])
+            original_path = original_uri.path.to_s
+            filename = format("%s-%s-%s", timestamp, File.basename(original_path, File.extname(original_path)), @comment_seq)
 
-              @original_url_base = "#{original_uri.scheme}://#{original_uri.host}"
-            else
-              raise "Original URL is missing"
-            end
+            @comment_seq += 1
+            @original_url_base = "#{original_uri.scheme}://#{original_uri.host}"
 
             header = {
               "date"            => @in_entry_elem[:meta][:published],
@@ -291,8 +279,8 @@ module JekyllImport
             body = @in_entry_elem[:body]
 
             # body escaping associated with liquid
-            body.gsub!(%r!{{!, '{{ "{{" }}') if body =~ %r!{{!
-            body.gsub!(%r!{%!, '{{ "{%" }}') if body =~ %r!{%!
+            body.gsub!(%r!{{!, '{{ "{{" }}') if %r!{{!.match?(body)
+            body.gsub!(%r!{%!, '{{ "{%" }}') if %r!{%!.match?(body)
 
             { :filename => filename, :header => header, :body => body }
           end

data/lib/jekyll-import/importers/drupal6.rb

@@ -19,15 +19,17 @@ module JekyllImport
                        nr.teaser,
                        n.created,
                        n.status,
+                       ua.dst AS alias,
                        n.type,
                        GROUP_CONCAT( td.name SEPARATOR '|' ) AS 'tags'
-                FROM #{prefix}node_revisions AS nr,
+                FROM #{prefix}node_revisions AS nr, url_alias AS ua,
                      #{prefix}node AS n
                      LEFT OUTER JOIN #{prefix}term_node AS tn ON tn.nid = n.nid
                      LEFT OUTER JOIN #{prefix}term_data AS td ON tn.tid = td.tid
                 WHERE (#{types})
                   AND n.vid = nr.vid
-                GROUP BY n.nid
+                  AND  ua.src = CONCAT( 'node/', n.nid)
+                GROUP BY n.nid, ua.dst
 SQL
 
         query
@@ -44,9 +46,11 @@ SQL
 
         data = {
           "excerpt"    => summary,
-          "categories" => tags.split("|"),
+          "categories" => tags.split("|").uniq,
         }
 
+        data["permalink"] = "/" + sql_post_data[:alias] if sql_post_data[:alias]
+
         [data, content]
       end
     end

data/lib/jekyll-import/importers/drupal8.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "jekyll-import/importers/drupal_common"
+
+module JekyllImport
+  module Importers
+    class Drupal8 < Importer
+      include DrupalCommon
+      extend DrupalCommon::ClassMethods
+
+      def self.build_query(prefix, types, engine)
+        types = types.join("' OR n.type = '")
+        types = "n.type = '#{types}'"
+
+        tag_group = if engine == "postgresql"
+                      <<POSTGRESQL
+            (SELECT STRING_AGG(td.name, '|')
+            FROM #{prefix}taxonomy_term_field_data td, #{prefix}taxonomy_index ti
+            WHERE ti.tid = td.tid AND ti.nid = n.nid) AS tags
+POSTGRESQL
+                    else
+                      <<SQL
+            (SELECT GROUP_CONCAT(td.name SEPARATOR '|')
+            FROM #{prefix}taxonomy_term_field_data td, #{prefix}taxonomy_index ti
+            WHERE ti.tid = td.tid AND ti.nid = n.nid) AS 'tags'
+SQL
+                    end
+
+        query = <<QUERY
+                SELECT n.nid,
+                       n.title,
+                       nb.body_value,
+                       nb.body_summary,
+                       n.created,
+                       n.status,
+                       n.type,
+                       #{tag_group}
+                FROM #{prefix}node_field_data AS n
+                LEFT JOIN #{prefix}node__body AS nb
+                  ON nb.entity_id = n.nid
+                WHERE (#{types})
+QUERY
+
+        query
+      end
+
+      def self.aliases_query(prefix)
+        "SELECT source, alias FROM #{prefix}url_alias WHERE source = ?"
+      end
+
+      def self.post_data(sql_post_data)
+        content = sql_post_data[:body_value].to_s
+        summary = sql_post_data[:body_summary].to_s
+        tags = (sql_post_data[:tags] || "").downcase.strip
+
+        data = {
+          "excerpt"    => summary,
+          "categories" => tags.split("|"),
+        }
+
+        [data, content]
+      end
+    end
+  end
+end

data/lib/jekyll-import/importers/drupal_common.rb

@@ -68,6 +68,7 @@ module JekyllImport
           src_dir = conf["source"]
 
           dirs = {
+            :_aliases => src_dir,
             :_posts   => File.join(src_dir, "_posts").to_s,
             :_drafts  => File.join(src_dir, "_drafts").to_s,
             :_layouts => Jekyll.sanitized_path(src_dir, conf["layouts_dir"].to_s),
@@ -146,10 +147,10 @@ module JekyllImport
 
               if partition.first.length.positive?
                 dir = "#{partition.first}/"
-                FileUtils.mkdir_p partition.first
+                FileUtils.mkdir_p "#{dirs[:_aliases]}/#{dir}"
               end
 
-              File.open("#{dir}#{file}.md", "w") do |f|
+              File.open("#{dirs[:_aliases]}/#{dir}#{file}.md", "w") do |f|
                 f.puts "---"
                 f.puts "layout: refresh"
                 f.puts "permalink: #{dir}#{file}/"

data/lib/jekyll-import/importers/marley.rb

@@ -48,7 +48,7 @@ module JekyllImport
           title = file_content.scan(regexp[:title]).first.to_s.strip
           prerex = file_content.scan(regexp[:perex]).first.to_s.strip
           published_on = DateTime.parse(post[:published_on]) rescue File.mtime(File.dirname(f))
-          meta          = meta_content ? YAML.safe_load(meta_content.scan(regexp[:meta]).to_s) : {}
+          meta = meta_content ? YAML.safe_load(meta_content.scan(regexp[:meta]).to_s) : {}
           meta["title"] = title
           meta["layout"] = "post"
 

data/lib/jekyll-import/importers/mephisto.rb

@@ -31,7 +31,7 @@ module JekyllImport
           rubygems
           sequel
           mysql2
-          fastercsv
+          csv
           fileutils
         ))
       end

data/lib/jekyll-import/importers/mt.rb

@@ -69,7 +69,7 @@ module JekyllImport
       # comments::        If true, output comments in _comments directory, like the one
       #                   mentioned at https://github.com/mpalmer/jekyll-static-comments/
       def self.process(options)
-        options  = default_options.merge(options)
+        options = default_options.merge(options)
 
         comments = options.fetch("comments")
         posts_name_by_id = {} if comments

data/lib/jekyll-import/importers/pluxml.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module JekyllImport
+  module Importers
+    class Pluxml < Importer
+      def self.require_deps
+        JekyllImport.require_with_fallback(%w(
+          nokogiri
+          fileutils
+          safe_yaml
+        ))
+      end
+
+      def self.specify_options(c)
+        c.option "source", "--source NAME", "The PluXML data directory to import"
+        c.option "layout", "--layout NAME", "The layout to apply"
+        c.option "avoid_liquid", "--avoid_liquid true", "Will add render_with_liquid: false in frontmatter"
+      end
+
+      def self.validate(options)
+        abort "Missing mandatory option --source." if options["source"].nil?
+        # no layout option, layout by default is post
+        options["layout"] = "post" if options["layout"].nil?
+        # no avoid_liquid option, avoid_liquid by default is false
+        options["avoid_liquid"] = false if options["avoid_liquid"].nil?
+      end
+
+      def self.process(options)
+        source       = options.fetch("source")
+        layout       = options.fetch("layout")
+        avoid_liquid = options.fetch("avoid_liquid")
+
+        FileUtils.mkdir_p("_posts")
+        FileUtils.mkdir_p("_drafts")
+
+        # for each XML file in source location
+        Dir.glob("*.xml", :base => source).each do |df|
+          df = File.join(source, df)
+          filename = File.basename(df, ".*")
+
+          # prepare post file name in Jekyll format
+          a_filename = filename.split(".")
+          post_name  = a_filename.pop
+          file_date  = a_filename.pop
+          post_date  = file_date[0..3] + "-" + file_date[4..5] + "-" + file_date[6..7]
+
+          # if draft, only take post name
+          if filename.split(".")[1].split(",")[0] == "draft"
+            directory = "_drafts"
+            name      = post_name.to_s
+          # if post, post date precede post name
+          else
+            directory = "_posts"
+            name      = "#{post_date}-#{post_name}"
+          end
+
+          xml = File.open(df) { |f| Nokogiri::XML(f) }
+          raise "There doesn't appear to be any XML items at the source (#{df}) provided." unless xml
+
+          doc = xml.xpath("document")
+          header = {
+            "layout" => layout,
+            "title"  => doc.xpath("title").text,
+            "tags"   => doc.xpath("tags").text.split(", "),
+          }
+          header["render_with_liquid"] = false if avoid_liquid
+
+          path = File.join(directory, "#{name}.html")
+          File.open(path, "w") do |f|
+            f.puts header.to_yaml
+            f.puts "---\n\n"
+            f.puts doc.xpath("chapo").text
+            f.puts doc.xpath("content").text
+          end
+
+          Jekyll.logger.info "Wrote file #{path} successfully!"
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/jekyll-import/importers/rss.rb

@@ -6,6 +6,7 @@ module JekyllImport
       def self.specify_options(c)
         c.option "source", "--source NAME", "The RSS file or URL to import"
         c.option "tag", "--tag NAME", "Add a tag to posts"
+        c.option "render_audio", "--render_audio", "Render <audio> element as necessary"
       end
 
       def self.validate(options)
@@ -30,8 +31,6 @@ module JekyllImport
       # Returns nothing.
       def self.process(options)
         source = options.fetch("source")
-        frontmatter = options.fetch("frontmatter", [])
-        body = options.fetch("body", ["description"])
 
         content = ""
         open(source) { |s| content = s.read }
@@ -40,37 +39,56 @@ module JekyllImport
         raise "There doesn't appear to be any RSS items at the source (#{source}) provided." unless rss
 
         rss.items.each do |item|
-          formatted_date = item.date.strftime("%Y-%m-%d")
-          post_name = Jekyll::Utils.slugify(item.title, :mode => "latin")
-          name = "#{formatted_date}-#{post_name}"
+          write_rss_item(item, options)
+        end
+      end
 
-          header = {
-            "layout" => "post",
-            "title"  => item.title,
-          }
+      def self.write_rss_item(item, options)
+        frontmatter = options.fetch("frontmatter", [])
+        body = options.fetch("body", ["description"])
+        render_audio = options.fetch("render_audio", false)
 
-          header["tag"] = options["tag"] unless options["tag"].nil? || options["tag"].empty?
+        formatted_date = item.date.strftime("%Y-%m-%d")
+        post_name = Jekyll::Utils.slugify(item.title, :mode => "latin")
+        name = "#{formatted_date}-#{post_name}"
+        audio = render_audio && item.enclosure.url
 
-          frontmatter.each do |value|
-            header[value] = item.send(value)
-          end
+        header = {
+          "layout" => "post",
+          "title"  => item.title,
+        }
 
-          output = +""
+        header["tag"] = options["tag"] unless options["tag"].nil? || options["tag"].empty?
 
-          body.each do |row|
-            output << item.send(row).to_s
-          end
+        frontmatter.each do |value|
+          header[value] = item.send(value)
+        end
 
-          output.strip!
-          output = item.content_encoded if output.empty?
+        output = +""
+
+        body.each do |row|
+          output << item.send(row).to_s
+        end
 
-          FileUtils.mkdir_p("_posts")
+        output.strip!
+        output = item.content_encoded if output.empty?
 
-          File.open("_posts/#{name}.html", "w") do |f|
-            f.puts header.to_yaml
-            f.puts "---\n\n"
-            f.puts output
+        FileUtils.mkdir_p("_posts")
+
+        File.open("_posts/#{name}.html", "w") do |f|
+          f.puts header.to_yaml
+          f.puts "---\n\n"
+
+          if audio
+            f.puts <<~HTML
+              <audio controls="">
+                <source src="#{audio}" type="audio/mpeg">
+                Your browser does not support the audio element.
+              </audio>
+            HTML
           end
+
+          f.puts output
         end
       end
     end

data/lib/jekyll-import/importers/s9y_database.rb

@@ -11,25 +11,31 @@ module JekyllImport
             fileutils
             safe_yaml
             unidecode
+            nokogiri
           )
         )
       end
 
       def self.specify_options(c)
-        c.option "dbname",         "--dbname DB",           "Database name (default: '')"
-        c.option "socket",         "--socket SOCKET",       "Database socket (default: '')"
-        c.option "user",           "--user USER",           "Database user name (default: '')"
-        c.option "password",       "--password PW",         "Database user's password (default: '')"
-        c.option "host",           "--host HOST",           "Database host name (default: 'localhost')"
-        c.option "port",           "--port PORT",           "Custom database port connect to (default: 3306)"
-        c.option "table_prefix",   "--table_prefix PREFIX", "Table prefix name (default: 'serendipity_')"
-        c.option "clean_entities", "--clean_entities",      "Whether to clean entities (default: true)"
-        c.option "comments",       "--comments",            "Whether to import comments (default: true)"
-        c.option "categories",     "--categories",          "Whether to import categories (default: true)"
-        c.option "tags",           "--tags",                "Whether to import tags (default: true)"
-        c.option "drafts",         "--drafts",              "Whether to export drafts as well"
-        c.option "markdown",       "--markdown",            "convert into markdown format (default: false)"
-        c.option "permalinks",     "--permalinks",          "preserve S9Y permalinks (default: false)"
+        c.option "dbname",            "--dbname DB",           "Database name (default: '')"
+        c.option "socket",            "--socket SOCKET",       "Database socket (default: '')"
+        c.option "user",              "--user USER",           "Database user name (default: '')"
+        c.option "password",          "--password PW",         "Database user's password (default: '')"
+        c.option "host",              "--host HOST",           "Database host name (default: 'localhost')"
+        c.option "port",              "--port PORT",           "Custom database port connect to (default: 3306)"
+        c.option "table_prefix",      "--table_prefix PREFIX", "Table prefix name (default: 'serendipity_')"
+        c.option "clean_entities",    "--clean_entities",      "Whether to clean entities (default: true)"
+        c.option "comments",          "--comments",            "Whether to import comments (default: true)"
+        c.option "categories",        "--categories",          "Whether to import categories (default: true)"
+        c.option "tags",              "--tags",                "Whether to import tags (default: true)"
+        c.option "drafts",            "--drafts",              "Whether to export drafts as well"
+        c.option "markdown",          "--markdown",            "convert into markdown format (default: false)"
+        c.option "permalinks",        "--permalinks",          "preserve S9Y permalinks (default: false)"
+        c.option "excerpt_separator", "--excerpt_separator",   "Demarkation for excerpts (default: '<a id=\"extended\"></a>')"
+        c.option "includeentry",      "--includeentry",        "Replace macros from the includeentry plugin (default: false)"
+        c.option "imgfig",            "--imgfig",              "Replace nested img and youtube divs with HTML figure tags (default: true)"
+        c.option "linebreak",         "--linebreak",           "Line break processing: wp, nokogiri, ignore (default: wp)"
+        c.option "relative",          "--relative",            "Convert links with this prefix to relative (default:nil)"
       end
 
       # Main migrator function. Call this to perform the migration.
@@ -56,36 +62,64 @@ module JekyllImport
       # :categories::     If true, save the post's categories in its
       #                   YAML front matter. Default: true.
       # :tags::           If true, save the post's tags in its
-      #                   YAML front matter. Default: true.
+      #                   YAML front matter, in lowercase.  Default: true.
       # :extension::      Set the post extension. Default: "html"
       # :drafts::         If true, export drafts as well
       #                   Default: true.
       # :markdown::       If true, convert the content to markdown
       #                   Default: false
       # :permalinks::     If true, save the post's original permalink in its
-      #                   YAML front matter. Default: false.
+      #                   YAML front matter. If the 'entryproperties' plugin
+      #                   was used, its permalink will become the canonical
+      #                   permalink, and any other will become redirects.
+      #                   Default: false.
+      # :excerpt_separator:: A string to use to separate the excerpt (body
+      #                      in S9Y) from the rest of the article (extended
+      #                      body in S9Y). Default: "<a id=\"extended\"></a>".
+      # :includentry::    Replace macros from the includentry plugin - these are
+      #                   the [s9y-include-entry] and [s9y-include-block] macros.
+      #                   Default: false.
+      # :imgfig::         Replace S9Y image-comment divs with an HTML figure
+      #                   div and figcaption, if applicable. Works for img and
+      #                   iframe.
+      #                   Default: true.
       #
+      # :linebreak::      When set to the default "wp", line breaks in entries
+      #                   will be processed WordPress style, by replacing double
+      #                   line breaks with HTML p tags, and remaining single
+      #                   line breaks with HTML br tags. When set to "nokogiri",
+      #                   entries will be loaded into Nokogiri and formatted as
+      #                   an XHTML fragment. When set to "ignore", line breaks
+      #                   will not be replaced at all.
+      #                   Default: wp
+      # :relative::       Replace absolute links (http://:relative:/foo)
+      #                   to relative links (/foo).
+
       def self.process(opts)
         options = {
-          :user           => opts.fetch("user", ""),
-          :pass           => opts.fetch("password", ""),
-          :host           => opts.fetch("host", "localhost"),
-          :port           => opts.fetch("port", 3306),
-          :socket         => opts.fetch("socket", nil),
-          :dbname         => opts.fetch("dbname", ""),
-          :table_prefix   => opts.fetch("table_prefix", "serendipity_"),
-          :clean_entities => opts.fetch("clean_entities", true),
-          :comments       => opts.fetch("comments", true),
-          :categories     => opts.fetch("categories", true),
-          :tags           => opts.fetch("tags", true),
-          :extension      => opts.fetch("extension", "html"),
-          :drafts         => opts.fetch("drafts", true),
-          :markdown       => opts.fetch("markdown", false),
-          :permalinks     => opts.fetch("permalinks", false),
+          :user              => opts.fetch("user", ""),
+          :pass              => opts.fetch("password", ""),
+          :host              => opts.fetch("host", "localhost"),
+          :port              => opts.fetch("port", 3306),
+          :socket            => opts.fetch("socket", nil),
+          :dbname            => opts.fetch("dbname", ""),
+          :table_prefix      => opts.fetch("table_prefix", "serendipity_"),
+          :clean_entities    => opts.fetch("clean_entities", true),
+          :comments          => opts.fetch("comments", true),
+          :categories        => opts.fetch("categories", true),
+          :tags              => opts.fetch("tags", true),
+          :extension         => opts.fetch("extension", "html"),
+          :drafts            => opts.fetch("drafts", true),
+          :markdown          => opts.fetch("markdown", false),
+          :permalinks        => opts.fetch("permalinks", false),
+          :excerpt_separator => opts.fetch("excerpt_separator", "<a id=\"extended\"></a>"),
+          :includeentry      => opts.fetch("includeentry", false),
+          :imgfig            => opts.fetch("imgfig", true),
+          :linebreak         => opts.fetch("linebreak", "wp"),
+          :relative          => opts.fetch("relative", nil),
         }
 
         options[:clean_entities] = require_if_available("htmlentities", "clean_entities") if options[:clean_entities]
-
         options[:markdown] = require_if_available("reverse_markdown", "markdown") if options[:markdown]
 
         FileUtils.mkdir_p("_posts")
@@ -120,6 +154,7 @@ module JekyllImport
 
         posts_query = "
            SELECT
+             'post'                 AS `type`,
              entries.ID             AS `id`,
              entries.isdraft        AS `isdraft`,
              entries.title          AS `title`,
@@ -154,36 +189,41 @@ module JekyllImport
         name = format("%02d-%02d-%02d-%s.%s", date.year, date.month, date.day, slug, extension)
 
         content = post[:body].to_s
-        content += "\n\n" + post[:body_extended].to_s unless post[:body_extended].to_s.empty?
+        extended_content = post[:body_extended].to_s
 
+        content += options[:excerpt_separator] + extended_content unless extended_content.nil? || extended_content.strip.empty?
+
+        content = process_includeentry(content, db, options) if options[:includeentry]
+        content = process_img_div(content) if options[:imgfig]
         content = clean_entities(content) if options[:clean_entities]
+        content = content.gsub(%r!href=(["'])http://#{options[:relative]}!, 'href=\1') if options[:relative]
 
         content = ReverseMarkdown.convert(content) if options[:markdown]
 
         categories = process_categories(db, options, post)
         comments = process_comments(db, options, post)
         tags = process_tags(db, options, post)
-        permalink = process_permalink(db, options, post)
+        all_permalinks = process_permalink(db, options, post)
+        primary_permalink = all_permalinks.shift
+        supplemental_permalinks = all_permalinks unless all_permalinks.empty?
 
         # Get the relevant fields as a hash, delete empty fields and
         # convert to YAML for the header.
         data = {
-          "layout"       => post[:type].to_s,
-          "status"       => status.to_s,
-          "published"    => status.to_s == "draft" ? nil : (status.to_s == "published"),
-          "title"        => title.to_s,
-          "author"       => {
-            "display_name" => post[:author].to_s,
-            "login"        => post[:author_login].to_s,
-            "email"        => post[:author_email].to_s,
-          },
-          "author_login" => post[:author_login].to_s,
-          "author_email" => post[:author_email].to_s,
-          "date"         => date.to_s,
-          "permalink"    => options[:permalinks] ? permalink : nil,
-          "categories"   => options[:categories] ? categories : nil,
-          "tags"         => options[:tags] ? tags : nil,
-          "comments"     => options[:comments] ? comments : nil,
+          "layout"            => post[:type].to_s,
+          "status"            => status.to_s,
+          "published"         => status.to_s == "draft" ? nil : (status.to_s == "published"),
+          "title"             => title.to_s,
+          "author"            => post[:author].to_s,
+          "author_login"      => post[:author_login].to_s,
+          "author_email"      => post[:author_email].to_s,
+          "date"              => date.to_s,
+          "permalink"         => options[:permalinks] ? primary_permalink : nil,
+          "redirect_from"     => options[:permalinks] ? supplemental_permalinks : nil,
+          "categories"        => options[:categories] ? categories : nil,
+          "tags"              => options[:tags] ? tags : nil,
+          "comments"          => options[:comments] ? comments : nil,
+          "excerpt_separator" => extended_content.empty? ? nil : options[:excerpt_separator],
         }.delete_if { |_k, v| v.nil? || v == "" }.to_yaml
 
         if post[:type] == "page"
@@ -195,11 +235,21 @@ module JekyllImport
           filename = "_posts/#{name}"
         end
 
+        content = case options[:linebreak]
+                  when "nokogiri"
+                    Nokogiri::HTML.fragment(content).to_xhtml
+                  when "ignore"
+                    content
+                  else
+                    # "wp" is the only remaining option, and the default
+                    Util.wpautop(content)
+                  end
+
         # Write out the data and content to file
         File.open(filename, "w") do |f|
           f.puts data
           f.puts "---"
-          f.puts Util.wpautop(content)
+          f.puts content
         end
       end
 
@@ -207,10 +257,154 @@ module JekyllImport
         require gem_name
         true
       rescue LoadError
-        warn "Could not require '#{gem_name}', so the :#{option_name} option is now disabled."
+        Jekyll.logger.warn "s9y database:", "Could not require '#{gem_name}', so the :#{option_name} option is now disabled."
         true
       end
 
+      def self.process_includeentry(text, db, options)
+        return text unless options[:includeentry]
+
+        result = text
+
+        px = options[:table_prefix]
+
+        props  = text.scan(%r!(\[s9y-include-entry:([0-9]+):([^:]+)\])!)
+        blocks = text.scan(%r!(\[s9y-include-block:([0-9]+):?([^:]+)?\])!)
+
+        props.each do |match|
+          macro = match[0]
+          id = match[1]
+          replacement = ""
+          if match[2].start_with?("prop=")
+            prop = match[2].sub("prop=", "")
+            cquery = get_property_query(px, id, prop)
+          else
+            prop = match[2]
+            cquery = get_value_query(px, id, prop)
+          end
+          db[cquery].each do |row|
+            replacement << row[:txt]
+          end
+          result = result.sub(macro, replacement)
+        end
+
+        blocks.each do |match|
+          macro = match[0]
+          id = match[1]
+          replacement = ""
+          # match[2] *could* be 'template', but we can't run it through Smarty, so we ignore it
+          cquery = %(
+            SELECT
+              px.body AS `txt`
+            FROM
+              #{px}staticblocks AS px
+            WHERE
+              id = '#{id}'
+          )
+          db[cquery].each do |row|
+            replacement << row[:txt]
+          end
+          result = result.sub(macro, replacement)
+        end
+
+        result
+      end
+
+      def get_property_query(px, id, prop)
+        %(
+          SELECT
+            px.value AS `txt`
+          FROM
+            #{px}entryproperties AS px
+          WHERE
+            entryid = '#{id}' AND
+            property = '#{prop}'
+        )
+      end
+
+      def get_value_query(px, id, prop)
+        %(
+          SELECT
+            px.#{prop} AS `txt`
+          FROM
+            #{px}entries AS px
+          WHERE
+            entryid = '#{id}'
+        )
+      end
+
+      # Replace .serendipity_imageComment_* blocks
+      def self.process_img_div(text)
+        caption_classes = [
+          ".serendipity_imageComment_left",
+          ".serendipity_imageComment_right",
+          ".serendipity_imageComment_center",
+        ]
+
+        noko = Nokogiri::HTML.fragment(text)
+        noko.css(caption_classes.join(",")).each do |imgcaption|
+          block_attrs = get_block_attrs(imgcaption)
+
+          # Is this a thumbnail to a bigger/other image?
+          big_link = imgcaption.at_css(".serendipity_image_link")
+          big_link ||= imgcaption.at_xpath(".//a[.//img]")
+
+          # The caption (if any) may have raw HTML
+          caption_elem = imgcaption.at_css(".serendipity_imageComment_txt")
+          caption = ""
+          caption = "<figcaption>#{caption_elem.inner_html}</figcaption>" if caption_elem
+
+          image_node = imgcaption.at_css("img")
+          if image_node
+            attrs = get_media_attrs(image_node)
+            media = "<img #{attrs}/>"
+          else
+            iframe_node = imgcaption.at_css("iframe")
+            if iframe_node
+              attrs = get_media_attrs(iframe_node)
+              media = "<iframe #{attrs}'></iframe>"
+            else
+              Jekyll.logger.warn "s9y database:", "Unrecognized media block: #{imgcaption}"
+              return text
+            end
+          end
+
+          # Wrap media in link, if any
+          if big_link
+            big = big_link.attribute("href")
+            media = "<a href='#{big}'>#{media}</a>"
+          end
+
+          # Replace HTML with clean media source, wrapped in figure
+          imgcaption.replace("<figure #{block_attrs}#{media}#{caption}</figure>")
+        end
+
+        noko.to_s
+      end
+
+      def get_media_attrs(node)
+        width = node.attribute("width")
+        width = "width='#{width}'" if width
+        height = node.attribute("height")
+        height = "height='#{height}'" if height
+        alt = node.attribute("alt")
+        alt = "alt='#{alt}'" if alt
+        src = "src='" + node.attribute("src") + "'"
+        [src, width, height, alt].join(" ")
+      end
+
+      def get_block_attrs(imgcaption)
+        # Extract block-level attributes
+        float = imgcaption.attribute("class").value.sub("serendipity_imageComment_", "")
+        float = "class='figure-#{float}'"
+        style = imgcaption.attribute("style")
+        style = " style='#{style.value}'" if style
+        # Don't lose good data
+        mdbnum = imgcaption.search(".//comment()").text.strip.sub("s9ymdb:", "")
+        mdb = "<!-- mdb='#{mdbnum}' -->" if mdbnum
+        [float, style, mdb].join(" ")
+      end
+
       def self.process_categories(db, options, post)
         return [] unless options[:categories]
 
@@ -278,7 +472,7 @@ module JekyllImport
       end
 
       def self.process_tags(db, options, post)
-        return [] unless options[:categories]
+        return [] unless options[:tags]
 
         px = options[:table_prefix]
 
@@ -293,18 +487,36 @@ module JekyllImport
 
         db[cquery].each_with_object([]) do |tag, tags|
           tags << if options[:clean_entities]
-                    clean_entities(tag[:name])
+                    clean_entities(tag[:name]).downcase
                   else
-                    tag[:name]
+                    tag[:name].downcase
                   end
         end
       end
 
       def self.process_permalink(db, options, post)
-        return unless options[:permalinks]
+        return [] unless options[:permalinks]
+
+        permalinks = []
 
         px = options[:table_prefix]
 
+        if db.table_exists?("#{px}entryproperties")
+          pquery = %(
+            SELECT
+              props.value AS `permalink`
+            FROM
+              #{px}entryproperties AS props
+            WHERE
+              props.entryid = '#{post[:id]}' AND
+              props.property = 'permalink'
+          )
+          db[pquery].each do |link|
+            plink = link[:permalink].to_s
+            permalinks << plink unless plink.end_with? "/UNKNOWN.html"
+          end
+        end
+
         cquery = %(
             SELECT
                permalinks.permalink AS `permalink`
@@ -316,8 +528,10 @@ module JekyllImport
         )
 
         db[cquery].each do |link|
-          return "/#{link[:permalink]}"
+          permalinks << "/#{link[:permalink]}"
         end
+
+        permalinks
       end
 
       def self.clean_entities(text)

data/lib/jekyll-import/importers/tumblr.rb

@@ -266,7 +266,7 @@ module JekyllImport
               lang  = "python"
               start = i
             elsif block
-              lang  = "javascript" if line =~ %r!;$!
+              lang  = "javascript" if %r!;$!.match?(line)
               block = line =~ indent && i < lines.size - 1 # Also handle EOF
               unless block
                 lines[start] = "{% highlight #{lang} %}"
@@ -283,7 +283,7 @@ module JekyllImport
           return url unless @grab_images
 
           path = "tumblr_files/#{url.split("/").last}"
-          path += ext unless path =~ %r!#{ext}$!
+          path += ext unless %r!#{ext}$!.match?(path)
           FileUtils.mkdir_p "tumblr_files"
 
           # Don't fetch if we've already cached this file

data/lib/jekyll-import/importers/typo.rb

@@ -55,7 +55,7 @@ module JekyllImport
           raise "Unknown database server '#{server}'"
         end
         db[SQL].each do |post|
-          next unless post[:state] =~ %r!published!i
+          next unless %r!published!i.match?(post[:state])
 
           post[:slug] = "no slug" if post[:slug].nil?
 
@@ -77,8 +77,8 @@ module JekyllImport
 
           File.open("_posts/#{name}", "w") do |f|
             f.puts({ "layout"  => "post",
-                     "title"   => (post[:title]&.to_s&.force_encoding("UTF-8")),
-                     "tags"    => (post[:keywords]&.to_s&.force_encoding("UTF-8")),
+                     "title"   => post[:title]&.to_s&.force_encoding("UTF-8"),
+                     "tags"    => post[:keywords]&.to_s&.force_encoding("UTF-8"),
                      "typo_id" => post[:id], }.delete_if { |_k, v| v.nil? || v == "" }.to_yaml)
             f.puts "---"
             f.puts post[:body].delete("\r")

data/lib/jekyll-import/importers/wordpressdotcom.rb

@@ -30,14 +30,15 @@ module JekyllImport
         images.each do |i|
           uri = i["src"]
 
-          i["src"] = format("{{ site.baseurl }}/%s/%s", assets_folder, File.basename(uri))
           dst = File.join(assets_folder, File.basename(uri))
+          i["src"] = File.join("{{ site.baseurl }}", dst)
           Jekyll.logger.info uri
           if File.exist?(dst)
             Jekyll.logger.info "Already in cache. Clean assets folder if you want a redownload."
             next
           end
           begin
+            FileUtils.mkdir_p assets_folder
             OpenURI.open_uri(uri, :allow_redirections => :safe) do |f|
               File.open(dst, "wb") do |out|
                 out.puts f.read
@@ -191,7 +192,16 @@ module JekyllImport
             content = Hpricot(item.text_for("content:encoded"))
             header["excerpt"] = item.excerpt if item.excerpt
 
-            download_images(item.title, content, assets_folder) if fetch
+            if fetch
+              # Put the images into a /yyyy/mm/ subfolder to reduce clashes
+              assets_dir_path = if item.published_at
+                                  File.join(assets_folder, item.published_at.strftime("/%Y/%m"))
+                                else
+                                  assets_folder
+                                end
+
+              download_images(item.title, content, assets_dir_path)
+            end
 
             FileUtils.mkdir_p item.directory_name
             File.open(File.join(item.directory_name, item.file_name), "w") do |f|

data/lib/jekyll-import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JekyllImport
-  VERSION = "0.19.0"
+  VERSION = "0.21.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-import
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.21.0
 platform: ruby
 authors:
 - Tom Preston-Werner
@@ -10,22 +10,8 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-24 00:00:00.000000000 Z
+date: 2021-11-01 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: fastercsv
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: jekyll
   requirement: !ruby/object:Gem::Requirement
@@ -74,20 +60,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.2'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -164,14 +136,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 0.11.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 0.11.0
 - !ruby/object:Gem::Dependency
   name: shoulda
   requirement: !ruby/object:Gem::Requirement
@@ -373,6 +345,7 @@ files:
 - lib/jekyll-import/importers/dotclear.rb
 - lib/jekyll-import/importers/drupal6.rb
 - lib/jekyll-import/importers/drupal7.rb
+- lib/jekyll-import/importers/drupal8.rb
 - lib/jekyll-import/importers/drupal_common.rb
 - lib/jekyll-import/importers/easyblog.rb
 - lib/jekyll-import/importers/enki.rb
@@ -384,6 +357,7 @@ files:
 - lib/jekyll-import/importers/marley.rb
 - lib/jekyll-import/importers/mephisto.rb
 - lib/jekyll-import/importers/mt.rb
+- lib/jekyll-import/importers/pluxml.rb
 - lib/jekyll-import/importers/posterous.rb
 - lib/jekyll-import/importers/roller.rb
 - lib/jekyll-import/importers/rss.rb
@@ -410,14 +384,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Import command for Jekyll (static site generator).