RubyGems - bunto-import - Versions diffs - 2.0.0 → 3.0.0 - Mend

bunto-import 2.0.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (35) hide show

checksums.yaml +4 -4
data/LICENSE +21 -21
data/README.markdown +33 -33
data/lib/bunto-import.rb +49 -49
data/lib/bunto-import/importer.rb +26 -26
data/lib/bunto-import/importers.rb +10 -10
data/lib/bunto-import/importers/behance.rb +80 -80
data/lib/bunto-import/importers/blogger.rb +330 -264
data/lib/bunto-import/importers/csv.rb +96 -96
data/lib/bunto-import/importers/drupal6.rb +53 -139
data/lib/bunto-import/importers/drupal7.rb +54 -111
data/lib/bunto-import/importers/drupal_common.rb +157 -0
data/lib/bunto-import/importers/easyblog.rb +96 -96
data/lib/bunto-import/importers/enki.rb +74 -74
data/lib/bunto-import/importers/ghost.rb +68 -68
data/lib/bunto-import/importers/google_reader.rb +64 -64
data/lib/bunto-import/importers/joomla.rb +92 -90
data/lib/bunto-import/importers/joomla3.rb +91 -91
data/lib/bunto-import/importers/jrnl.rb +125 -125
data/lib/bunto-import/importers/marley.rb +72 -72
data/lib/bunto-import/importers/mephisto.rb +99 -99
data/lib/bunto-import/importers/mt.rb +257 -257
data/lib/bunto-import/importers/posterous.rb +130 -130
data/lib/bunto-import/importers/rss.rb +62 -62
data/lib/bunto-import/importers/s9y.rb +60 -60
data/lib/bunto-import/importers/s9y_database.rb +363 -0
data/lib/bunto-import/importers/textpattern.rb +70 -70
data/lib/bunto-import/importers/tumblr.rb +300 -289
data/lib/bunto-import/importers/typo.rb +88 -88
data/lib/bunto-import/importers/wordpress.rb +372 -372
data/lib/bunto-import/importers/wordpressdotcom.rb +207 -207
data/lib/bunto-import/util.rb +76 -76
data/lib/bunto-import/version.rb +3 -3
data/lib/bunto/commands/import.rb +79 -79
metadata +84 -54

data/lib/bunto-import/importers/drupal_common.rb ADDED

@@ -0,0 +1,157 @@
+require 'date'
+module BuntoImport
+  module Importers
+    module DrupalCommon
+      # This module provides a base for the Drupal importers (at least for 6
+      # and 7; since 8 will be a different beast). Version-specific importers
+      # will need to implement the missing methods from the Importer class.
+      #
+      # The general idea is that this importer reads a MySQL database via Sequel
+      # and creates a post file for each node it finds in the Drupal database.
+      module ClassMethods
+        DEFAULTS = {
+          "password" => "",
+          "host"     => "localhost",
+          "prefix"   => "",
+          "types"    => %w(blog story article)
+        }
+        def specify_options(c)
+          c.option 'dbname', '--dbname DB', 'Database name'
+          c.option 'user', '--user USER', 'Database user name'
+          c.option 'password', '--password PW', "Database user's password (default: #{DEFAULTS["password"].inspect})"
+          c.option 'host', '--host HOST', "Database host name (default: #{DEFAULTS["host"].inspect})"
+          c.option 'prefix', '--prefix PREFIX', "Table prefix name (default: #{DEFAULTS["prefix"].inspect})"
+          c.option 'types', '--types TYPE1[,TYPE2[,TYPE3...]]', Array,
+            "The Drupal content types to be imported  (default: #{DEFAULTS["types"].join(",")})"
+        end
+        def require_deps
+          BuntoImport.require_with_fallback(%w[
+            rubygems
+            sequel
+            fileutils
+            safe_yaml
+          ])
+        end
+        def process(options)
+          dbname = options.fetch('dbname')
+          user   = options.fetch('user')
+          pass   = options.fetch('password', DEFAULTS["password"])
+          host   = options.fetch('host',     DEFAULTS["host"])
+          prefix = options.fetch('prefix',   DEFAULTS["prefix"])
+          types  = options.fetch('types',    DEFAULTS["types"])
+          db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+          query = self.build_query(prefix, types)
+          conf = Bunto.configuration({})
+          src_dir = conf['source']
+          dirs = {
+              :_posts   => File.join(src_dir, '_posts').to_s,
+              :_drafts  => File.join(src_dir, '_drafts').to_s,
+              :_layouts => Bunto.sanitized_path(src_dir, conf['layouts_dir'].to_s)
+          }
+          dirs.each do |key, dir|
+            FileUtils.mkdir_p dir
+          end
+          # Create the refresh layout
+          # Change the refresh url if you customized your permalink config
+          File.open(File.join(dirs[:_layouts], 'refresh.html'), 'w') do |f|
+            f.puts <<-HTML
+<!DOCTYPE html>
+<html>
+<head>
+<meta http-equiv="content-type" content="text/html; charset=utf-8" />
+<meta http-equiv="refresh" content="0;url={{ page.refresh_to_post_id }}.html" />
+</head>
+</html>
+HTML
+          end
+          db[query].each do |post|
+            # Get required fields
+            data, content = self.post_data(post)
+            data['layout'] = post[:type]
+            title = data['title'] = post[:title].strip.force_encoding('UTF-8')
+            time = data['created'] = post[:created]
+            # Get the relevant fields as a hash and delete empty fields
+            data = data.delete_if { |k,v| v.nil? || v == ''}.each_pair {
+                |k,v| ((v.is_a? String) ? v.force_encoding('UTF-8') : v)
+            }
+            # Construct a Bunto compatible file name
+            is_published = post[:status] == 1
+            node_id = post[:nid]
+            dir = is_published ? dirs[:_posts] : dirs[:_drafts]
+            slug = title.strip.downcase.gsub(/(&|&amp;)/, ' and ').gsub(/[\s\.\/\\]/, '-').gsub(/[^\w-]/, '').gsub(/[-_]{2,}/, '-').gsub(/^[-_]/, '').gsub(/[-_]$/, '')
+            filename = Time.at(time).to_datetime.strftime('%Y-%m-%d-') + slug + '.md'
+            # Write out the data and content to file
+            File.open("#{dir}/#{filename}", 'w') do |f|
+              f.puts data.to_yaml
+              f.puts '---'
+              f.puts content
+            end
+            # Make a file to redirect from the old Drupal URL
+            if is_published
+              alias_query = self.aliases_query(prefix)
+              type = post[:type]
+              aliases = db[alias_query, "#{type}/#{node_id}"].all
+              aliases.push(:alias => "#{type}/#{node_id}")
+              aliases.each do |url_alias|
+                FileUtils.mkdir_p url_alias[:alias]
+                File.open("#{url_alias[:alias]}/index.md", "w") do |f|
+                  f.puts '---'
+                  f.puts 'layout: refresh'
+                  f.puts "refresh_to_post_id: /#{Time.at(time).to_datetime.strftime('%Y/%m/%d/') + slug}"
+                  f.puts '---'
+                end
+              end
+            end
+          end
+        end
+      end
+      def build_query(prefix, types)
+        raise 'The importer you are trying to use does not implement the get_query() method.'
+      end
+      def aliases_query(prefix)
+        # Make sure you implement the query returning "alias" as the column name
+        # for the URL aliases. See the Drupal 6 importer for an example. The
+        # alias field is called 'dst' but we alias it to 'alias', to follow
+        # Drupal 7's column names.
+        raise 'The importer you are trying to use does not implement the get_aliases_query() method.'
+      end
+      def post_data(sql_post_data)
+        raise 'The importer you are trying to use does not implement the get_query() method.'
+      end
+      def validate(options)
+        %w[dbname user].each do |option|
+          if options[option].nil?
+            abort "Missing mandatory option --#{option}."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bunto-import/importers/easyblog.rb CHANGED

@@ -1,96 +1,96 @@
-module BuntoImport
-  module Importers
-    class Easyblog < Importer
-      def self.validate(options)
-        %w[dbname user].each do |option|
-          if options[option].nil?
-            abort "Missing mandatory option --#{option}."
-          end
-        end
-      end
-      def self.specify_options(c)
-        c.option 'dbname', '--dbname', 'Database name'
-        c.option 'user', '--user', 'Database user name'
-        c.option 'password', '--password', "Database user's password (default: '')"
-        c.option 'host', '--host', 'Database host name'
-        c.option 'section', '--section', 'Table prefix name'
-        c.option 'prefix', '--prefix', 'Table prefix name'
-      end
-      def self.require_deps
-        BuntoImport.require_with_fallback(%w[
-                                           rubygems
-                                          sequel
-                                          fileutils
-                                          safe_yaml
-                                          ])
-      end
-      def self.process(options)
-        dbname  = options.fetch('dbname')
-        user    = options.fetch('user')
-        pass    = options.fetch('password', '')
-        host    = options.fetch('host', "localhost")
-        section = options.fetch('section', '1')
-        table_prefix = options.fetch('prefix', "jos_")
-        db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
-        FileUtils.mkdir_p("_posts")
-        # Reads a MySQL database via Sequel and creates a post file for each
-        # post in wp_posts that has post_status = 'publish'. This restriction is
-        # made because 'draft' posts are not guaranteed to have valid dates.
-        query = "
-        select
-	  ep.`title`, `permalink` as alias, concat(`intro`, `content`) as content, ep.`created`, ep.`id`, ec.`title` as category, tags
-        from
-          #{table_prefix}easyblog_post ep
-          left join #{table_prefix}easyblog_category ec on (ep.category_id = ec.id)
-          left join (
-            select
-              ept.post_id,
-              group_concat(et.alias order by alias separator ' ') as tags
-            from
-              #{table_prefix}easyblog_post_tag ept
-              join #{table_prefix}easyblog_tag et on (ept.tag_id = et.id)
-            group by
-              ept.post_id) x on (ep.id = x.post_id);
-        "
-        db[query].each do |post|
-          # Get required fields and construct Bunto compatible name.
-          title = post[:title]
-          slug = post[:alias]
-          date = post[:created]
-          content = post[:content]
-          category = post[:category]
-          tags = post[:tags]
-          name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day,
-                                                 slug]
-          # Get the relevant fields as a hash, delete empty fields and convert
-          # to YAML for the header.
-          data = {
-            'layout' => 'post',
-            'title' => title.to_s,
-            'joomla_id' => post[:id],
-            'joomla_url' => post[:alias],
-            'category' => post[:category],
-            'tags' => post[:tags],
-            'date' => date
-          }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
-          # Write out the data and content to file
-          File.open("_posts/#{name}", "w") do |f|
-            f.puts data
-            f.puts "---"
-            f.puts content
-          end
-        end
-      end
-    end
-  end
-end
+module BuntoImport
+  module Importers
+    class Easyblog < Importer
+      def self.validate(options)
+        %w[dbname user].each do |option|
+          if options[option].nil?
+            abort "Missing mandatory option --#{option}."
+          end
+        end
+      end
+      def self.specify_options(c)
+        c.option 'dbname', '--dbname', 'Database name'
+        c.option 'user', '--user', 'Database user name'
+        c.option 'password', '--password', "Database user's password (default: '')"
+        c.option 'host', '--host', 'Database host name'
+        c.option 'section', '--section', 'Table prefix name'
+        c.option 'prefix', '--prefix', 'Table prefix name'
+      end
+      def self.require_deps
+        BuntoImport.require_with_fallback(%w[
+                                           rubygems
+                                          sequel
+                                          fileutils
+                                          safe_yaml
+                                          ])
+      end
+      def self.process(options)
+        dbname  = options.fetch('dbname')
+        user    = options.fetch('user')
+        pass    = options.fetch('password', '')
+        host    = options.fetch('host', "localhost")
+        section = options.fetch('section', '1')
+        table_prefix = options.fetch('prefix', "jos_")
+        db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+        FileUtils.mkdir_p("_posts")
+        # Reads a MySQL database via Sequel and creates a post file for each
+        # post in wp_posts that has post_status = 'publish'. This restriction is
+        # made because 'draft' posts are not guaranteed to have valid dates.
+        query = "
+        select
+	  ep.`title`, `permalink` as alias, concat(`intro`, `content`) as content, ep.`created`, ep.`id`, ec.`title` as category, tags
+        from
+          #{table_prefix}easyblog_post ep
+          left join #{table_prefix}easyblog_category ec on (ep.category_id = ec.id)
+          left join (
+            select
+              ept.post_id,
+              group_concat(et.alias order by alias separator ' ') as tags
+            from
+              #{table_prefix}easyblog_post_tag ept
+              join #{table_prefix}easyblog_tag et on (ept.tag_id = et.id)
+            group by
+              ept.post_id) x on (ep.id = x.post_id);
+        "
+        db[query].each do |post|
+          # Get required fields and construct Bunto compatible name.
+          title = post[:title]
+          slug = post[:alias]
+          date = post[:created]
+          content = post[:content]
+          category = post[:category]
+          tags = post[:tags]
+          name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day,
+                                                 slug]
+          # Get the relevant fields as a hash, delete empty fields and convert
+          # to YAML for the header.
+          data = {
+            'layout' => 'post',
+            'title' => title.to_s,
+            'joomla_id' => post[:id],
+            'joomla_url' => post[:alias],
+            'category' => post[:category],
+            'tags' => post[:tags],
+            'date' => date
+          }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
+          # Write out the data and content to file
+          File.open("_posts/#{name}", "w") do |f|
+            f.puts data
+            f.puts "---"
+            f.puts content
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bunto-import/importers/enki.rb CHANGED

@@ -1,74 +1,74 @@
-module BuntoImport
-    module Importers
-    class Enki < Importer
-      SQL = <<-EOS
-        SELECT p.id,
-               p.title,
-               p.slug,
-               p.body,
-               p.published_at as date,
-               p.cached_tag_list as tags
-        FROM posts p
-EOS
-      def self.validate(options)
-        %w[dbname user].each do |option|
-          if options[option].nil?
-            abort "Missing mandatory option --#{option}."
-          end
-        end
-      end
-      def self.specify_options(c)
-        c.option 'dbname', '--dbname', 'Database name'
-        c.option 'user', '--user', 'Database name'
-        c.option 'password', '--password', 'Database name (default: "")'
-        c.option 'host', '--host', 'Database name'
-      end
-      def self.require_deps
-        BuntoImport.require_with_fallback(%w[
-          rubygems
-          sequel
-          fileutils
-          pg
-          yaml
-        ])
-      end
-      # Just working with postgres, but can be easily adapted
-      # to work with both mysql and postgres.
-      def self.process(options)
-        dbname = options.fetch('dbname')
-        user   = options.fetch('user')
-        pass   = options.fetch('password', "")
-        host   = options.fetch('host', "localhost")
-        FileUtils.mkdir_p('_posts')
-        db = Sequel.postgres(:database => dbname,
-                             :user => user,
-                             :password => pass,
-                             :host => host,
-                             :encoding => 'utf8')
-        db[SQL].each do |post|
-          name = [ sprintf("%.04d", post[:date].year),
-                   sprintf("%.02d", post[:date].month),
-                   sprintf("%.02d", post[:date].day),
-                   post[:slug].strip ].join('-')
-          name += '.textile'
-          File.open("_posts/#{name}", 'w') do |f|
-            f.puts({ 'layout'   => 'post',
-                     'title'    => post[:title].to_s,
-                     'enki_id'  => post[:id],
-                     'categories'  => post[:tags]
-                   }.delete_if { |k, v| v.nil? || v == '' }.to_yaml)
-            f.puts '---'
-            f.puts post[:body].delete("\r")
-          end
-        end
-      end
-    end
-  end
-end
+module BuntoImport
+    module Importers
+    class Enki < Importer
+      SQL = <<-EOS
+        SELECT p.id,
+               p.title,
+               p.slug,
+               p.body,
+               p.published_at as date,
+               p.cached_tag_list as tags
+        FROM posts p
+EOS
+      def self.validate(options)
+        %w[dbname user].each do |option|
+          if options[option].nil?
+            abort "Missing mandatory option --#{option}."
+          end
+        end
+      end
+      def self.specify_options(c)
+        c.option 'dbname', '--dbname', 'Database name'
+        c.option 'user', '--user', 'Database name'
+        c.option 'password', '--password', 'Database name (default: "")'
+        c.option 'host', '--host', 'Database name'
+      end
+      def self.require_deps
+        BuntoImport.require_with_fallback(%w[
+          rubygems
+          sequel
+          fileutils
+          pg
+          yaml
+        ])
+      end
+      # Just working with postgres, but can be easily adapted
+      # to work with both mysql and postgres.
+      def self.process(options)
+        dbname = options.fetch('dbname')
+        user   = options.fetch('user')
+        pass   = options.fetch('password', "")
+        host   = options.fetch('host', "localhost")
+        FileUtils.mkdir_p('_posts')
+        db = Sequel.postgres(:database => dbname,
+                             :user => user,
+                             :password => pass,
+                             :host => host,
+                             :encoding => 'utf8')
+        db[SQL].each do |post|
+          name = [ sprintf("%.04d", post[:date].year),
+                   sprintf("%.02d", post[:date].month),
+                   sprintf("%.02d", post[:date].day),
+                   post[:slug].strip ].join('-')
+          name += '.textile'
+          File.open("_posts/#{name}", 'w') do |f|
+            f.puts({ 'layout'   => 'post',
+                     'title'    => post[:title].to_s,
+                     'enki_id'  => post[:id],
+                     'categories'  => post[:tags]
+                   }.delete_if { |k, v| v.nil? || v == '' }.to_yaml)
+            f.puts '---'
+            f.puts post[:body].delete("\r")
+          end
+        end
+      end
+    end
+  end
+end