bunto-import 2.0.0 → 3.0.0

data/lib/bunto-import/importers/wordpressdotcom.rb

@@ -1,207 +1,207 @@
-# encoding: UTF-8
-
-module BuntoImport
-  module Importers
-    class WordpressDotCom < Importer
-      def self.require_deps
-        BuntoImport.require_with_fallback(%w[
-          rubygems
-          fileutils
-          safe_yaml
-          hpricot
-          time
-          open-uri
-          open_uri_redirections
-        ])
-      end
-
-      def self.specify_options(c)
-        c.option 'source', '--source FILE', 'WordPress export XML file (default: "wordpress.xml")'
-        c.option 'no_fetch_images', '--no-fetch-images', 'Do not fetch the images referenced in the posts'
-        c.option 'assets_folder', '--assets_folder FOLDER', 'Folder where assets such as images will be downloaded to (default: assets)'
-      end
-
-      # Will modify post DOM tree
-      def self.download_images(title, post_hpricot, assets_folder)
-        images = (post_hpricot/"img")
-        if images.length == 0
-          return
-        end
-        puts "Downloading images for " + title
-        images.each do |i|
-          uri = i["src"]
-
-          i["src"] = "{{ site.baseurl }}/%s/%s" % [assets_folder, File.basename(uri)]
-          dst = File.join(assets_folder, File.basename(uri))
-          puts "  " + uri
-          if File.exist?(dst)
-            puts "    Already in cache. Clean assets folder if you want a redownload."
-            next
-          end
-          begin
-            open(uri, allow_redirections: :safe) {|f|
-              File.open(dst, "wb") do |out|
-                out.puts f.read
-              end
-            }
-            puts "    OK!"
-          rescue => e
-            puts "    Error: #{e.message}"
-            puts e.backtrace.join("\n")
-          end
-        end
-      end
-
-      class Item
-        def initialize(node)
-          @node = node
-        end
-
-        def text_for(path)
-          @node.at(path).inner_text
-        end
-
-        def title
-          @title ||= text_for(:title).strip
-        end
-
-        def permalink_title
-          post_name = text_for('wp:post_name')
-          # Fallback to "prettified" title if post_name is empty (can happen)
-          @permalink_title ||= if post_name.empty?
-            WordpressDotCom.sluggify(title)
-          else
-            post_name
-          end
-        end
-
-        def published_at
-          if published?
-            @published_at ||= Time.parse(text_for('wp:post_date'))
-          end
-        end
-
-        def status
-          @status ||= text_for('wp:status')
-        end
-
-        def post_type
-          @post_type ||= text_for('wp:post_type')
-        end
-
-        def file_name
-          @file_name ||= if published?
-            "#{published_at.strftime('%Y-%m-%d')}-#{permalink_title}.html"
-          else
-            "#{permalink_title}.html"
-          end
-        end
-
-        def directory_name
-          @directory_name ||= if !published? && post_type == 'post'
-            '_drafts'
-          else
-            "_#{post_type}s"
-          end
-        end
-
-        def published?
-          @published ||= (status == 'publish')
-        end
-
-        def excerpt
-          @excerpt ||= begin
-            text = Hpricot(text_for('excerpt:encoded')).inner_text
-            if text.empty?
-              nil
-            else
-              text
-            end
-          end
-        end
-      end
-
-      def self.process(options)
-        source        = options.fetch('source', "wordpress.xml")
-        fetch         = !options.fetch('no_fetch_images', false)
-        assets_folder = options.fetch('assets_folder', 'assets')
-        FileUtils.mkdir_p(assets_folder)
-
-        import_count = Hash.new(0)
-        doc = Hpricot::XML(File.read(source))
-        # Fetch authors data from header
-        authors = Hash[
-          (doc/:channel/'wp:author').map do |author|
-          [author.at("wp:author_login").inner_text.strip, {
-            "login" => author.at("wp:author_login").inner_text.strip,
-            "email" => author.at("wp:author_email").inner_text,
-            "display_name" => author.at("wp:author_display_name").inner_text,
-            "first_name" => author.at("wp:author_first_name").inner_text,
-            "last_name" => author.at("wp:author_last_name").inner_text
-          }]
-          end
-        ] rescue {}
-
-        (doc/:channel/:item).each do |node|
-          item = Item.new(node)
-          categories = node.search('category[@domain="category"]').map(&:inner_text).reject{|c| c == 'Uncategorized'}.uniq
-          tags = node.search('category[@domain="post_tag"]').map(&:inner_text).uniq
-
-          metas = Hash.new
-          node.search("wp:postmeta").each do |meta|
-            key = meta.at('wp:meta_key').inner_text
-            value = meta.at('wp:meta_value').inner_text
-            metas[key] = value
-          end
-
-          author_login = item.text_for('dc:creator').strip
-
-          header = {
-            'layout'     => item.post_type,
-            'title'      => item.title,
-            'date'       => item.published_at,
-            'type'       => item.post_type,
-            'published'  => item.published?,
-            'status'     => item.status,
-            'categories' => categories,
-            'tags'       => tags,
-            'meta'       => metas,
-            'author'     => authors[author_login]
-          }
-
-          begin
-            content = Hpricot(item.text_for('content:encoded'))
-            header['excerpt'] = item.excerpt if item.excerpt
-
-            if fetch
-              download_images(item.title, content, assets_folder)
-            end
-
-            FileUtils.mkdir_p item.directory_name
-            File.open(File.join(item.directory_name, item.file_name), "w") do |f|
-              f.puts header.to_yaml
-              f.puts '---'
-              f.puts Util.wpautop(content.to_html)
-            end
-          rescue => e
-            puts "Couldn't import post!"
-            puts "Title: #{item.title}"
-            puts "Name/Slug: #{item.file_name}\n"
-            puts "Error: #{e.message}"
-            next
-          end
-
-          import_count[item.post_type] += 1
-        end
-
-        import_count.each do |key, value|
-          puts "Imported #{value} #{key}s"
-        end
-      end
-
-      def self.sluggify(title)
-        title.gsub(/[^[:alnum:]]+/, '-').downcase
-      end
-    end
-  end
-end
+# encoding: UTF-8
+
+module BuntoImport
+  module Importers
+    class WordpressDotCom < Importer
+      def self.require_deps
+        BuntoImport.require_with_fallback(%w[
+          rubygems
+          fileutils
+          safe_yaml
+          hpricot
+          time
+          open-uri
+          open_uri_redirections
+        ])
+      end
+
+      def self.specify_options(c)
+        c.option 'source', '--source FILE', 'WordPress export XML file (default: "wordpress.xml")'
+        c.option 'no_fetch_images', '--no-fetch-images', 'Do not fetch the images referenced in the posts'
+        c.option 'assets_folder', '--assets_folder FOLDER', 'Folder where assets such as images will be downloaded to (default: assets)'
+      end
+
+      # Will modify post DOM tree
+      def self.download_images(title, post_hpricot, assets_folder)
+        images = (post_hpricot/"img")
+        if images.length == 0
+          return
+        end
+        puts "Downloading images for " + title
+        images.each do |i|
+          uri = i["src"]
+
+          i["src"] = "{{ site.baseurl }}/%s/%s" % [assets_folder, File.basename(uri)]
+          dst = File.join(assets_folder, File.basename(uri))
+          puts "  " + uri
+          if File.exist?(dst)
+            puts "    Already in cache. Clean assets folder if you want a redownload."
+            next
+          end
+          begin
+            open(uri, allow_redirections: :safe) {|f|
+              File.open(dst, "wb") do |out|
+                out.puts f.read
+              end
+            }
+            puts "    OK!"
+          rescue => e
+            puts "    Error: #{e.message}"
+            puts e.backtrace.join("\n")
+          end
+        end
+      end
+
+      class Item
+        def initialize(node)
+          @node = node
+        end
+
+        def text_for(path)
+          @node.at(path).inner_text
+        end
+
+        def title
+          @title ||= text_for(:title).strip
+        end
+
+        def permalink_title
+          post_name = text_for('wp:post_name')
+          # Fallback to "prettified" title if post_name is empty (can happen)
+          @permalink_title ||= if post_name.empty?
+            WordpressDotCom.sluggify(title)
+          else
+            post_name
+          end
+        end
+
+        def published_at
+          if published?
+            @published_at ||= Time.parse(text_for('wp:post_date'))
+          end
+        end
+
+        def status
+          @status ||= text_for('wp:status')
+        end
+
+        def post_type
+          @post_type ||= text_for('wp:post_type')
+        end
+
+        def file_name
+          @file_name ||= if published?
+            "#{published_at.strftime('%Y-%m-%d')}-#{permalink_title}.html"
+          else
+            "#{permalink_title}.html"
+          end
+        end
+
+        def directory_name
+          @directory_name ||= if !published? && post_type == 'post'
+            '_drafts'
+          else
+            "_#{post_type}s"
+          end
+        end
+
+        def published?
+          @published ||= (status == 'publish')
+        end
+
+        def excerpt
+          @excerpt ||= begin
+            text = Hpricot(text_for('excerpt:encoded')).inner_text
+            if text.empty?
+              nil
+            else
+              text
+            end
+          end
+        end
+      end
+
+      def self.process(options)
+        source        = options.fetch('source', "wordpress.xml")
+        fetch         = !options.fetch('no_fetch_images', false)
+        assets_folder = options.fetch('assets_folder', 'assets')
+        FileUtils.mkdir_p(assets_folder)
+
+        import_count = Hash.new(0)
+        doc = Hpricot::XML(File.read(source))
+        # Fetch authors data from header
+        authors = Hash[
+          (doc/:channel/'wp:author').map do |author|
+          [author.at("wp:author_login").inner_text.strip, {
+            "login" => author.at("wp:author_login").inner_text.strip,
+            "email" => author.at("wp:author_email").inner_text,
+            "display_name" => author.at("wp:author_display_name").inner_text,
+            "first_name" => author.at("wp:author_first_name").inner_text,
+            "last_name" => author.at("wp:author_last_name").inner_text
+          }]
+          end
+        ] rescue {}
+
+        (doc/:channel/:item).each do |node|
+          item = Item.new(node)
+          categories = node.search('category[@domain="category"]').map(&:inner_text).reject{|c| c == 'Uncategorized'}.uniq
+          tags = node.search('category[@domain="post_tag"]').map(&:inner_text).uniq
+
+          metas = Hash.new
+          node.search("wp:postmeta").each do |meta|
+            key = meta.at('wp:meta_key').inner_text
+            value = meta.at('wp:meta_value').inner_text
+            metas[key] = value
+          end
+
+          author_login = item.text_for('dc:creator').strip
+
+          header = {
+            'layout'     => item.post_type,
+            'title'      => item.title,
+            'date'       => item.published_at,
+            'type'       => item.post_type,
+            'published'  => item.published?,
+            'status'     => item.status,
+            'categories' => categories,
+            'tags'       => tags,
+            'meta'       => metas,
+            'author'     => authors[author_login]
+          }
+
+          begin
+            content = Hpricot(item.text_for('content:encoded'))
+            header['excerpt'] = item.excerpt if item.excerpt
+
+            if fetch
+              download_images(item.title, content, assets_folder)
+            end
+
+            FileUtils.mkdir_p item.directory_name
+            File.open(File.join(item.directory_name, item.file_name), "w") do |f|
+              f.puts header.to_yaml
+              f.puts '---'
+              f.puts Util.wpautop(content.to_html)
+            end
+          rescue => e
+            puts "Couldn't import post!"
+            puts "Title: #{item.title}"
+            puts "Name/Slug: #{item.file_name}\n"
+            puts "Error: #{e.message}"
+            next
+          end
+
+          import_count[item.post_type] += 1
+        end
+
+        import_count.each do |key, value|
+          puts "Imported #{value} #{key}s"
+        end
+      end
+
+      def self.sluggify(title)
+        title.gsub(/[^[:alnum:]]+/, '-').downcase
+      end
+    end
+  end
+end

data/lib/bunto-import/util.rb

@@ -1,76 +1,76 @@
-module BuntoImport
-  module Util
-
-    # Ruby translation of wordpress wpautop (see https://core.trac.wordpress.org/browser/trunk/src/wp-includes/formatting.php)
-    #
-    # A group of regex replaces used to identify text formatted with newlines and
-    # replace double line-breaks with HTML paragraph tags. The remaining
-    # line-breaks after conversion become <<br />> tags, unless $br is set to false
-    #
-    # @param string pee The text which has to be formatted.
-    # @param bool br Optional. If set, this will convert all remaining line-breaks after paragraphing. Default true.
-    # @return string Text which has been converted into correct paragraph tags.
-    #
-    def self.wpautop(pee, br = true)
-      return '' if pee.strip == ''
-
-      allblocks = '(?:table|thead|tfoot|caption|col|colgroup|tbody|tr|td|th|div|dl|dd|dt|ul|ol|li|pre|select|option|form|map|area|blockquote|address|math|style|p|h[1-6]|hr|fieldset|noscript|legend|section|article|aside|hgroup|header|footer|nav|figure|figcaption|details|menu|summary)'
-      pre_tags = {}
-      pee = pee + "\n"
-
-      if pee.include?('<pre')
-        pee_parts = pee.split('</pre>')
-        last_pee = pee_parts.pop
-        pee = ''
-        pee_parts.each_with_index do |pee_part, i|
-          start = pee_part.index('<pre')
-
-          unless start
-            pee += pee_part
-            next
-          end
-
-          name = "<pre wp-pre-tag-#{i}></pre>"
-          pre_tags[name] = pee_part[start..-1] + '</pre>'
-
-          pee += pee_part[0, start] + name
-        end
-        pee += last_pee
-      end
-
-      pee = pee.gsub(Regexp.new('<br />\s*<br />'), "\n\n")
-      pee = pee.gsub(Regexp.new("(<" + allblocks + "[^>]*>)"), "\n\\1")
-      pee = pee.gsub(Regexp.new("(</" + allblocks + ">)"), "\\1\n\n")
-      pee = pee.gsub("\r\n", "\n").gsub("\r", "\n")
-      if pee.include? '<object'
-        pee = pee.gsub(Regexp.new('\s*<param([^>]*)>\s*'), "<param\\1>")
-        pee = pee.gsub(Regexp.new('\s*</embed>\s*'), '</embed>')
-      end
-
-      pees = pee.split(/\n\s*\n/).compact
-      pee = ''
-      pees.each { |tinkle| pee += '<p>' + tinkle.chomp("\n") + "</p>\n" }
-      pee = pee.gsub(Regexp.new('<p>\s*</p>'), '')
-      pee = pee.gsub(Regexp.new('<p>([^<]+)</(div|address|form)>'), "<p>\\1</p></\\2>")
-      pee = pee.gsub(Regexp.new('<p>\s*(</?' + allblocks + '[^>]*>)\s*</p>'), "\\1")
-      pee = pee.gsub(Regexp.new('<p>(<li.+?)</p>'), "\\1")
-      pee = pee.gsub(Regexp.new('<p><blockquote([^>]*)>', 'i'), "<blockquote\\1><p>")
-      pee = pee.gsub('</blockquote></p>', '</p></blockquote>')
-      pee = pee.gsub(Regexp.new('<p>\s*(</?' + allblocks + '[^>]*>)'), "\\1")
-      pee = pee.gsub(Regexp.new('(</?' + allblocks + '[^>]*>)\s*</p>'), "\\1")
-      if br
-        pee = pee.gsub(Regexp.new('<(script|style).*?</\1>')) { |match| match.gsub("\n", "<WPPreserveNewline />") }
-        pee = pee.gsub(Regexp.new('(?<!<br />)\s*\n'), "<br />\n")
-        pee = pee.gsub('<WPPreserveNewline />', "\n")
-      end
-      pee = pee.gsub(Regexp.new('(</?' + allblocks + '[^>]*>)\s*<br />'), "\\1")
-      pee = pee.gsub(Regexp.new('<br />(\s*</?(?:p|li|div|dl|dd|dt|th|pre|td|ul|ol)[^>]*>)'), "\\1")
-      pee = pee.gsub(Regexp.new('\n</p>$'), '</p>')
-
-      pre_tags.each do |name, value|
-        pee.gsub!(name, value)
-      end
-      pee
-    end
-  end
-end
+module BuntoImport
+  module Util
+
+    # Ruby translation of wordpress wpautop (see https://core.trac.wordpress.org/browser/trunk/src/wp-includes/formatting.php)
+    #
+    # A group of regex replaces used to identify text formatted with newlines and
+    # replace double line-breaks with HTML paragraph tags. The remaining
+    # line-breaks after conversion become <<br />> tags, unless $br is set to false
+    #
+    # @param string pee The text which has to be formatted.
+    # @param bool br Optional. If set, this will convert all remaining line-breaks after paragraphing. Default true.
+    # @return string Text which has been converted into correct paragraph tags.
+    #
+    def self.wpautop(pee, br = true)
+      return '' if pee.strip == ''
+
+      allblocks = '(?:table|thead|tfoot|caption|col|colgroup|tbody|tr|td|th|div|dl|dd|dt|ul|ol|li|pre|select|option|form|map|area|blockquote|address|math|style|p|h[1-6]|hr|fieldset|noscript|legend|section|article|aside|hgroup|header|footer|nav|figure|figcaption|details|menu|summary)'
+      pre_tags = {}
+      pee = pee + "\n"
+
+      if pee.include?('<pre')
+        pee_parts = pee.split('</pre>')
+        last_pee = pee_parts.pop
+        pee = ''
+        pee_parts.each_with_index do |pee_part, i|
+          start = pee_part.index('<pre')
+
+          unless start
+            pee += pee_part
+            next
+          end
+
+          name = "<pre wp-pre-tag-#{i}></pre>"
+          pre_tags[name] = pee_part[start..-1] + '</pre>'
+
+          pee += pee_part[0, start] + name
+        end
+        pee += last_pee
+      end
+
+      pee = pee.gsub(Regexp.new('<br />\s*<br />'), "\n\n")
+      pee = pee.gsub(Regexp.new("(<" + allblocks + "[^>]*>)"), "\n\\1")
+      pee = pee.gsub(Regexp.new("(</" + allblocks + ">)"), "\\1\n\n")
+      pee = pee.gsub("\r\n", "\n").gsub("\r", "\n")
+      if pee.include? '<object'
+        pee = pee.gsub(Regexp.new('\s*<param([^>]*)>\s*'), "<param\\1>")
+        pee = pee.gsub(Regexp.new('\s*</embed>\s*'), '</embed>')
+      end
+
+      pees = pee.split(/\n\s*\n/).compact
+      pee = ''
+      pees.each { |tinkle| pee += '<p>' + tinkle.chomp("\n") + "</p>\n" }
+      pee = pee.gsub(Regexp.new('<p>\s*</p>'), '')
+      pee = pee.gsub(Regexp.new('<p>([^<]+)</(div|address|form)>'), "<p>\\1</p></\\2>")
+      pee = pee.gsub(Regexp.new('<p>\s*(</?' + allblocks + '[^>]*>)\s*</p>'), "\\1")
+      pee = pee.gsub(Regexp.new('<p>(<li.+?)</p>'), "\\1")
+      pee = pee.gsub(Regexp.new('<p><blockquote([^>]*)>', 'i'), "<blockquote\\1><p>")
+      pee = pee.gsub('</blockquote></p>', '</p></blockquote>')
+      pee = pee.gsub(Regexp.new('<p>\s*(</?' + allblocks + '[^>]*>)'), "\\1")
+      pee = pee.gsub(Regexp.new('(</?' + allblocks + '[^>]*>)\s*</p>'), "\\1")
+      if br
+        pee = pee.gsub(Regexp.new('<(script|style).*?</\1>')) { |match| match.gsub("\n", "<WPPreserveNewline />") }
+        pee = pee.gsub(Regexp.new('(?<!<br />)\s*\n'), "<br />\n")
+        pee = pee.gsub('<WPPreserveNewline />', "\n")
+      end
+      pee = pee.gsub(Regexp.new('(</?' + allblocks + '[^>]*>)\s*<br />'), "\\1")
+      pee = pee.gsub(Regexp.new('<br />(\s*</?(?:p|li|div|dl|dd|dt|th|pre|td|ul|ol)[^>]*>)'), "\\1")
+      pee = pee.gsub(Regexp.new('\n</p>$'), '</p>')
+
+      pre_tags.each do |name, value|
+        pee.gsub!(name, value)
+      end
+      pee
+    end
+  end
+end