bunto-import 1.0.0

data/lib/bunto-import/importers/wordpressdotcom.rb

@@ -0,0 +1,207 @@
+# encoding: UTF-8
+
+module BuntoImport
+  module Importers
+    class WordpressDotCom < Importer
+      def self.require_deps
+        BuntoImport.require_with_fallback(%w[
+          rubygems
+          fileutils
+          safe_yaml
+          hpricot
+          time
+          open-uri
+          open_uri_redirections
+        ])
+      end
+
+      def self.specify_options(c)
+        c.option 'source', '--source FILE', 'WordPress export XML file (default: "wordpress.xml")'
+        c.option 'no_fetch_images', '--no-fetch-images', 'Do not fetch the images referenced in the posts'
+        c.option 'assets_folder', '--assets_folder FOLDER', 'Folder where assets such as images will be downloaded to (default: assets)'
+      end
+
+      # Will modify post DOM tree
+      def self.download_images(title, post_hpricot, assets_folder)
+        images = (post_hpricot/"img")
+        if images.length == 0
+          return
+        end
+        puts "Downloading images for " + title
+        images.each do |i|
+          uri = i["src"]
+
+          i["src"] = "{{ site.baseurl }}/%s/%s" % [assets_folder, File.basename(uri)]
+          dst = File.join(assets_folder, File.basename(uri))
+          puts "  " + uri
+          if File.exist?(dst)
+            puts "    Already in cache. Clean assets folder if you want a redownload."
+            next
+          end
+          begin
+            open(uri, allow_redirections: :safe) {|f|
+              File.open(dst, "wb") do |out|
+                out.puts f.read
+              end
+            }
+            puts "    OK!"
+          rescue => e
+            puts "    Error: #{e.message}"
+            puts e.backtrace.join("\n")
+          end
+        end
+      end
+
+      class Item
+        def initialize(node)
+          @node = node
+        end
+
+        def text_for(path)
+          @node.at(path).inner_text
+        end
+
+        def title
+          @title ||= text_for(:title).strip
+        end
+
+        def permalink_title
+          post_name = text_for('wp:post_name')
+          # Fallback to "prettified" title if post_name is empty (can happen)
+          @permalink_title ||= if post_name.empty?
+            WordpressDotCom.sluggify(title)
+          else
+            post_name
+          end
+        end
+
+        def published_at
+          if published?
+            @published_at ||= Time.parse(text_for('wp:post_date'))
+          end
+        end
+
+        def status
+          @status ||= text_for('wp:status')
+        end
+
+        def post_type
+          @post_type ||= text_for('wp:post_type')
+        end
+
+        def file_name
+          @file_name ||= if published?
+            "#{published_at.strftime('%Y-%m-%d')}-#{permalink_title}.html"
+          else
+            "#{permalink_title}.html"
+          end
+        end
+
+        def directory_name
+          @directory_name ||= if !published? && post_type == 'post'
+            '_drafts'
+          else
+            "_#{post_type}s"
+          end
+        end
+
+        def published?
+          @published ||= (status == 'publish')
+        end
+
+        def excerpt
+          @excerpt ||= begin
+            text = Hpricot(text_for('excerpt:encoded')).inner_text
+            if text.empty?
+              nil
+            else
+              text
+            end
+          end
+        end
+      end
+
+      def self.process(options)
+        source        = options.fetch('source', "wordpress.xml")
+        fetch         = !options.fetch('no_fetch_images', false)
+        assets_folder = options.fetch('assets_folder', 'assets')
+        FileUtils.mkdir_p(assets_folder)
+
+        import_count = Hash.new(0)
+        doc = Hpricot::XML(File.read(source))
+        # Fetch authors data from header
+        authors = Hash[
+          (doc/:channel/'wp:author').map do |author|
+          [author.at("wp:author_login").inner_text.strip, {
+            "login" => author.at("wp:author_login").inner_text.strip,
+            "email" => author.at("wp:author_email").inner_text,
+            "display_name" => author.at("wp:author_display_name").inner_text,
+            "first_name" => author.at("wp:author_first_name").inner_text,
+            "last_name" => author.at("wp:author_last_name").inner_text
+          }]
+          end
+        ] rescue {}
+
+        (doc/:channel/:item).each do |node|
+          item = Item.new(node)
+          categories = node.search('category[@domain="category"]').map(&:inner_text).reject{|c| c == 'Uncategorized'}.uniq
+          tags = node.search('category[@domain="post_tag"]').map(&:inner_text).uniq
+
+          metas = Hash.new
+          node.search("wp:postmeta").each do |meta|
+            key = meta.at('wp:meta_key').inner_text
+            value = meta.at('wp:meta_value').inner_text
+            metas[key] = value
+          end
+
+          author_login = item.text_for('dc:creator').strip
+
+          header = {
+            'layout'     => item.post_type,
+            'title'      => item.title,
+            'date'       => item.published_at,
+            'type'       => item.post_type,
+            'published'  => item.published?,
+            'status'     => item.status,
+            'categories' => categories,
+            'tags'       => tags,
+            'meta'       => metas,
+            'author'     => authors[author_login]
+          }
+
+          begin
+            content = Hpricot(item.text_for('content:encoded'))
+            header['excerpt'] = item.excerpt if item.excerpt
+
+            if fetch
+              download_images(item.title, content, assets_folder)
+            end
+
+            FileUtils.mkdir_p item.directory_name
+            File.open(File.join(item.directory_name, item.file_name), "w") do |f|
+              f.puts header.to_yaml
+              f.puts '---'
+              f.puts Util.wpautop(content.to_html)
+            end
+          rescue => e
+            puts "Couldn't import post!"
+            puts "Title: #{item.title}"
+            puts "Name/Slug: #{item.file_name}\n"
+            puts "Error: #{e.message}"
+            next
+          end
+
+          import_count[item.post_type] += 1
+        end
+
+        import_count.each do |key, value|
+          puts "Imported #{value} #{key}s"
+        end
+      end
+
+      def self.sluggify(title)
+        title.gsub(/[^[:alnum:]]+/, '-').downcase
+      end
+    end
+  end
+end

data/lib/bunto-import/util.rb

@@ -0,0 +1,76 @@
+module BuntoImport
+  module Util
+
+    # Ruby translation of wordpress wpautop (see https://core.trac.wordpress.org/browser/trunk/src/wp-includes/formatting.php)
+    #
+    # A group of regex replaces used to identify text formatted with newlines and
+    # replace double line-breaks with HTML paragraph tags. The remaining
+    # line-breaks after conversion become <<br />> tags, unless $br is set to false
+    #
+    # @param string pee The text which has to be formatted.
+    # @param bool br Optional. If set, this will convert all remaining line-breaks after paragraphing. Default true.
+    # @return string Text which has been converted into correct paragraph tags.
+    #
+    def self.wpautop(pee, br = true)
+      return '' if pee.strip == ''
+
+      allblocks = '(?:table|thead|tfoot|caption|col|colgroup|tbody|tr|td|th|div|dl|dd|dt|ul|ol|li|pre|select|option|form|map|area|blockquote|address|math|style|p|h[1-6]|hr|fieldset|noscript|legend|section|article|aside|hgroup|header|footer|nav|figure|figcaption|details|menu|summary)'
+      pre_tags = {}
+      pee = pee + "\n"
+
+      if pee.include?('<pre')
+        pee_parts = pee.split('</pre>')
+        last_pee = pee_parts.pop
+        pee = ''
+        pee_parts.each_with_index do |pee_part, i|
+          start = pee_part.index('<pre')
+
+          unless start
+            pee += pee_part
+            next
+          end
+
+          name = "<pre wp-pre-tag-#{i}></pre>"
+          pre_tags[name] = pee_part[start..-1] + '</pre>'
+
+          pee += pee_part[0, start] + name
+        end
+        pee += last_pee
+      end
+
+      pee = pee.gsub(Regexp.new('<br />\s*<br />'), "\n\n")
+      pee = pee.gsub(Regexp.new("(<" + allblocks + "[^>]*>)"), "\n\\1")
+      pee = pee.gsub(Regexp.new("(</" + allblocks + ">)"), "\\1\n\n")
+      pee = pee.gsub("\r\n", "\n").gsub("\r", "\n")
+      if pee.include? '<object'
+        pee = pee.gsub(Regexp.new('\s*<param([^>]*)>\s*'), "<param\\1>")
+        pee = pee.gsub(Regexp.new('\s*</embed>\s*'), '</embed>')
+      end
+
+      pees = pee.split(/\n\s*\n/).compact
+      pee = ''
+      pees.each { |tinkle| pee += '<p>' + tinkle.chomp("\n") + "</p>\n" }
+      pee = pee.gsub(Regexp.new('<p>\s*</p>'), '')
+      pee = pee.gsub(Regexp.new('<p>([^<]+)</(div|address|form)>'), "<p>\\1</p></\\2>")
+      pee = pee.gsub(Regexp.new('<p>\s*(</?' + allblocks + '[^>]*>)\s*</p>'), "\\1")
+      pee = pee.gsub(Regexp.new('<p>(<li.+?)</p>'), "\\1")
+      pee = pee.gsub(Regexp.new('<p><blockquote([^>]*)>', 'i'), "<blockquote\\1><p>")
+      pee = pee.gsub('</blockquote></p>', '</p></blockquote>')
+      pee = pee.gsub(Regexp.new('<p>\s*(</?' + allblocks + '[^>]*>)'), "\\1")
+      pee = pee.gsub(Regexp.new('(</?' + allblocks + '[^>]*>)\s*</p>'), "\\1")
+      if br
+        pee = pee.gsub(Regexp.new('<(script|style).*?</\1>')) { |match| match.gsub("\n", "<WPPreserveNewline />") }
+        pee = pee.gsub(Regexp.new('(?<!<br />)\s*\n'), "<br />\n")
+        pee = pee.gsub('<WPPreserveNewline />', "\n")
+      end
+      pee = pee.gsub(Regexp.new('(</?' + allblocks + '[^>]*>)\s*<br />'), "\\1")
+      pee = pee.gsub(Regexp.new('<br />(\s*</?(?:p|li|div|dl|dd|dt|th|pre|td|ul|ol)[^>]*>)'), "\\1")
+      pee = pee.gsub(Regexp.new('\n</p>$'), '</p>')
+
+      pre_tags.each do |name, value|
+        pee.gsub!(name, value)
+      end
+      pee
+    end
+  end
+end

data/lib/bunto-import/version.rb

@@ -0,0 +1,3 @@
+module BuntoImport
+  VERSION = '1.0.0'
+end

data/lib/bunto/commands/import.rb

@@ -0,0 +1,79 @@
+$:.unshift File.expand_path("../../", File.dirname(__FILE__)) # load from bunto-import/lib
+require 'bunto/command'
+require 'bunto-import'
+
+module Bunto
+  module Commands
+    class Import < Command
+
+      IMPORTERS = {
+        :blogger => 'Blogger',
+        :behance => 'Behance',
+        :csv => 'CSV',
+        :drupal6 => 'Drupal6',
+        :drupal7 => 'Drupal7',
+        :enki => 'Enki',
+        :joomla => 'Joomla',
+        :joomla3 => 'Joomla3',
+        :jrnl => 'Jrnl',
+        :ghost => 'Ghost',
+        :google_reader => 'GoogleReader',
+        :marley => 'Marley',
+        :mephisto => 'Mephisto',
+        :mt => 'MT',
+        :posterous => 'Posterous',
+        :rss => 'RSS',
+        :s9y => 'S9Y',
+        :textpattern => 'TextPattern',
+        :tumblr => 'Tumblr',
+        :typo => 'Typo',
+        :wordpress => 'WordPress',
+        :wordpressdotcom => 'WordpressDotCom'
+      }
+
+      class << self
+
+        def init_with_program(prog)
+          prog.command(:import) do |c|
+            c.syntax 'import <platform> [options]'
+            c.description 'Import your old blog to Bunto'
+            importers = BuntoImport.add_importer_commands(c)
+
+            c.action do |args, options|
+              if args.empty?
+                Bunto.logger.warn "You must specify an importer."
+                Bunto.logger.info "Valid options are:"
+                importers.each { |i| Bunto.logger.info "*", "#{i}" }
+              end
+            end
+          end
+        end
+
+        def process(migrator, options)
+          migrator = migrator.to_s.downcase
+
+          if IMPORTERS.keys.include?(migrator.to_sym)
+            if BuntoImport::Importers.const_defined?(IMPORTERS[migrator.to_sym])
+              klass = BuntoImport::Importers.const_get(IMPORTERS[migrator.to_sym])
+              if options.respond_to?(:__hash__)
+                klass.run(options.__hash__)
+              else
+                klass.run(options)
+              end
+            end
+          else
+            abort_on_invalid_migrator(migrator)
+          end
+        end
+
+        def abort_on_invalid_migrator(migrator)
+          $stderr.puts "Sorry, '#{migrator}' isn't a valid migrator. Valid choices:"
+          IMPORTERS.keys.each { |k| $stderr.puts "* #{k}" }
+          raise RuntimeError.new("'#{migrator}' is not a valid migrator.")
+        end
+
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,374 @@
+--- !ruby/object:Gem::Specification
+name: bunto-import
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Tom Preston-Werner
+- Suriyaa Kudo
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-04-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bunto
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: fastercsv
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: nokogiri
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.1.0
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.1.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  name: rdoc
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  name: activesupport
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  name: redgreen
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  name: shoulda
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: rr
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  name: simplecov
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  name: simplecov-gem-adapter
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.42'
+  name: sequel
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.42'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
+  name: htmlentities
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  name: hpricot
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  name: mysql
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  name: pg
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  name: mysql2
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  name: behance
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: unidecode
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: open_uri_redirections
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  name: launchy
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+description: Provides the Import command for Bunto.
+email:
+- tom@mojombo.com
+- SuriyaaKudoIsc@users.noreply.github.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.markdown
+- LICENSE
+files:
+- LICENSE
+- README.markdown
+- lib/bunto-import.rb
+- lib/bunto-import/importer.rb
+- lib/bunto-import/importers.rb
+- lib/bunto-import/importers/behance.rb
+- lib/bunto-import/importers/blogger.rb
+- lib/bunto-import/importers/csv.rb
+- lib/bunto-import/importers/drupal6.rb
+- lib/bunto-import/importers/drupal7.rb
+- lib/bunto-import/importers/easyblog.rb
+- lib/bunto-import/importers/enki.rb
+- lib/bunto-import/importers/ghost.rb
+- lib/bunto-import/importers/google_reader.rb
+- lib/bunto-import/importers/joomla.rb
+- lib/bunto-import/importers/joomla3.rb
+- lib/bunto-import/importers/jrnl.rb
+- lib/bunto-import/importers/marley.rb
+- lib/bunto-import/importers/mephisto.rb
+- lib/bunto-import/importers/mt.rb
+- lib/bunto-import/importers/posterous.rb
+- lib/bunto-import/importers/rss.rb
+- lib/bunto-import/importers/s9y.rb
+- lib/bunto-import/importers/textpattern.rb
+- lib/bunto-import/importers/tumblr.rb
+- lib/bunto-import/importers/typo.rb
+- lib/bunto-import/importers/wordpress.rb
+- lib/bunto-import/importers/wordpressdotcom.rb
+- lib/bunto-import/util.rb
+- lib/bunto-import/version.rb
+- lib/bunto/commands/import.rb
+homepage: http://github.com/bunto/bunto-import
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options:
+- "--charset=UTF-8"
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 2
+summary: Import command for Bunto (static site generator).
+test_files: []