jekyll-import 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5109d834df31bbc016f4fa9ed39b543cf7c914e
-  data.tar.gz: 6954fc663c0cd876b3704ddbfb22503d35777820
+  metadata.gz: 07d3bbdc383ff8bcf6b7b686332687345dc8a71b
+  data.tar.gz: 73fa392dc3e4957d0aae9dce535a0abb3655c73f
 SHA512:
-  metadata.gz: e4210991177bd53645dc452a866231e90bd7ecfb65832702683f053cf5ee66ce7e01133db3984d436941150408af49e9ba69c3b91cfd2c37c1080d2e2c797249
-  data.tar.gz: 80b9a2d9153efda4337f4245e8b6e7e01ff793b83659c4e149ebc7b40631d0327b46a6bb80f51effcf8da3c1a384899117e72938cb493062510145beb2e49e6e
+  metadata.gz: 32ddb7124f545ddc9031190529a40348051b713c2343f509ce26bc7970310ed456ed59e0dd7f4a7baaafde3141cd3248f42b85f47d3d26b237b090d452ade2d7
+  data.tar.gz: 6331a88ff5fe73131e7d4f0331e0767ff4871b190135260f42af846a2ea0b2f2c989ae678c0992400ac2defe3818a2d34b16fdf2bb3e26fc52a36a7b4af767ef

data/README.markdown

@@ -8,7 +8,7 @@ The new __Jekyll__ command for importing from various blogs to Jekyll format.
 
 ## How `jekyll-import` works:
 
-### Jekyll v2.x
+### Jekyll v2.x and higher
 
 1. Install the _rubygem_ with `gem install jekyll-import`.
 2. Run `jekyll import IMPORTER [options]`

data/lib/jekyll-import/importers/blogger.rb

@@ -24,6 +24,7 @@ module JekyllImport
           time
           fileutils
           safe_yaml
+          open-uri
         ])
       end
 
@@ -100,11 +101,11 @@ module JekyllImport
       module BloggerAtomStreamListenerMethods
         attr_accessor :leave_blogger_info
         attr_reader :original_url_base
-      
+
         def tag_start(tag, attrs)
           @tag_bread = [] unless @tag_bread
           @tag_bread.push(tag)
-      
+
           case tag
           when 'entry'
             raise 'nest entry element' if @in_entry_elem
@@ -144,7 +145,7 @@ module JekyllImport
             end
           end
         end
-      
+
         def text(text)
           if @in_entry_elem
             case @tag_bread.last
@@ -168,13 +169,13 @@ module JekyllImport
               end
             end
           end
-        end 
-      
+        end
+
         def tag_end(tag)
           case tag
           when 'entry'
             raise 'nest entry element' unless @in_entry_elem
-      
+
             if @in_entry_elem[:meta][:kind] == 'post'
               post_data = get_post_data_from_in_entry_elem_info
 
@@ -183,20 +184,21 @@ module JekyllImport
                 target_dir = '_drafts' if @in_entry_elem[:meta][:draft]
 
                 FileUtils.mkdir_p(target_dir)
-      
-                File.open(File.join(target_dir, "#{post_data[:filename]}.html"), 'w') do |f|
+
+                file_name = URI::decode("#{post_data[:filename]}.html")
+                File.open(File.join(target_dir, file_name), 'w') do |f|
                   f.flock(File::LOCK_EX)
-      
+
                   f << post_data[:header].to_yaml
                   f << "---\n\n"
                   f << post_data[:body]
                 end
               end
             end
-      
+
             @in_entry_elem = nil
           end
-      
+
           @tag_bread.pop
         end
 
@@ -223,10 +225,10 @@ module JekyllImport
                   [timestamp,
                    CGI.escape(name.downcase).tr('+','-')]
               end
-            else 
+            else
               raise 'Original URL is missing'
             end
-        
+
             header = {
               'layout' => 'post',
               'title' => @in_entry_elem[:meta][:title],
@@ -238,7 +240,7 @@ module JekyllImport
             header['thumbnail'] = @in_entry_elem[:meta][:thumbnail] if @in_entry_elem[:meta][:thumbnail]
             header['blogger_id'] = @in_entry_elem[:meta][:id] if @leave_blogger_info
             header['blogger_orig_url'] = @in_entry_elem[:meta][:original_url] if @leave_blogger_info && @in_entry_elem[:meta][:original_url]
-        
+
             body = @in_entry_elem[:body]
 
             # body escaping associated with liquid
@@ -248,7 +250,7 @@ module JekyllImport
             if body =~ /{%/
               body.gsub!(/{%/, '{{ "{%" }}')
             end
-  
+
             { :filename => filename, :header => header, :body => body }
           else
             nil

data/lib/jekyll-import/importers/joomla.rb

@@ -42,16 +42,25 @@ module JekyllImport
         # Reads a MySQL database via Sequel and creates a post file for each
         # post in wp_posts that has post_status = 'publish'. This restriction is
         # made because 'draft' posts are not guaranteed to have valid dates.
-        query = "SELECT `title`, `alias`, CONCAT(`introtext`,`fulltext`) as content, `created`, `id` FROM #{table_prefix}content WHERE state = '0' OR state = '1' AND sectionid = '#{section}'"
+        query = "SELECT `title`, `alias`, CONCAT(`introtext`,`fulltext`) as content, `created`, `id` FROM #{table_prefix}content WHERE (state = '0' OR state = '1') AND sectionid = '#{section}'"
 
         db[query].each do |post|
           # Get required fields and construct Jekyll compatible name.
           title = post[:title]
-          slug = post[:alias]
           date = post[:created]
           content = post[:content]
-          name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day,
-                                                 slug]
+          id = post[:id]
+
+          # Construct a slug from the title if alias field empty.
+          # Remove illegal filename characters.
+          if !post[:alias] or post[:alias].empty?
+            slug = sluggify(post[:title])
+          else
+            slug = sluggify(post[:alias])
+          end
+
+          name = "%02d-%02d-%02d-%03d-%s.markdown" % [date.year, date.month, date.day,
+                                                 id,slug]
 
           # Get the relevant fields as a hash, delete empty fields and convert
           # to YAML for the header.
@@ -71,6 +80,11 @@ module JekyllImport
           end
         end
       end
+
+      # Borrowed from the Wordpress importer
+      def self.sluggify( title )
+        title = title.downcase.gsub(/[^0-9A-Za-z]+/, " ").strip.gsub(" ", "-")
+      end
     end
   end
 end

data/lib/jekyll-import/importers/joomla3.rb

@@ -0,0 +1,91 @@
+module JekyllImport
+  module Importers
+    class Joomla3 < Importer
+      def self.validate(options)
+        %w[dbname user prefix].each do |option|
+          if options[option].nil?
+            abort "Missing mandatory option --#{option}."
+          end
+        end
+      end
+
+      def self.specify_options(c)
+        c.option 'dbname', '--dbname', 'Database name'
+        c.option 'user', '--user', 'Database user name'
+        c.option 'password', '--password', "Database user's password (default: '')"
+        c.option 'host', '--host', 'Database host name'
+        c.option 'category', '--category', 'ID of the category'
+        c.option 'prefix', '--prefix', 'Table prefix name'
+      end
+
+      def self.require_deps
+        JekyllImport.require_with_fallback(%w[
+          rubygems
+          sequel
+          fileutils
+          safe_yaml
+        ])
+      end
+
+      def self.process(options)
+        dbname  = options.fetch('dbname')
+        user    = options.fetch('user')
+        pass    = options.fetch('password', '')
+        host    = options.fetch('host', "localhost")
+        cid	    = options.fetch('category', 0)
+        table_prefix = options.fetch('prefix', "jos_")
+
+        db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
+
+        FileUtils.mkdir_p("_posts")
+
+        # Reads a MySQL database via Sequel and creates a post file for each
+        # post in #__content that is published.
+        query = "SELECT `cn`.`title`, `cn`.`alias`, `cn`.`introtext`, CONCAT(`cn`.`introtext`,`cn`.`fulltext`) AS `content`, "
+		query << "`cn`.`created`, `cn`.`id`, `ct`.`title` AS `category`, `u`.`name` AS `author` "
+		query << "FROM `#{table_prefix}content` AS `cn` JOIN `#{table_prefix}categories` AS `ct` ON `cn`.`catid` = `ct`.`id` "
+		query << "JOIN `#{table_prefix}users` AS `u` ON `cn`.`created_by` = `u`.`id` "
+		query << "WHERE (`cn`.`state` = '1' OR `cn`.`state` = '2') " # Only published and archived content items to be imported
+
+		if cid > 0
+			query << " AND `cn`.`catid` = '#{cid}' "
+		else 
+			query << " AND `cn`.`catid` != '2' " #Filter out uncategorized content
+		end
+
+        db[query].each do |post|
+          # Get required fields and construct Jekyll compatible name.
+          title = post[:title]
+          slug = post[:alias]
+          date = post[:created]
+		  author = post[:author]
+		  category = post[:category]
+          content = post[:content]
+		  excerpt = post[:introtext]
+          name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day,
+                                                 slug]
+
+          # Get the relevant fields as a hash, delete empty fields and convert
+          # to YAML for the header.
+          data = {
+             'layout' => 'post',
+             'title' => title.to_s,
+             'joomla_id' => post[:id],
+             'joomla_url' => slug,
+             'date' => date,
+			 'author' => author,
+			 'excerpt' => excerpt.strip.to_s,
+			 'category' => category
+           }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
+
+          # Write out the data and content to file
+          File.open("_posts/#{name}", "w") do |f|
+            f.puts data
+            f.puts "---"
+            f.puts content
+          end
+        end
+      end
+    end
+  end
+end

data/lib/jekyll-import/importers/mt.rb

@@ -26,10 +26,17 @@ module JekyllImport
       end
 
       def self.specify_options(c)
+        c.option 'engine', "--engine ENGINE", "Database engine, (default: 'mysql', postgres also supported)"
         c.option 'dbname', '--dbname DB', 'Database name'
         c.option 'user', '--user USER', 'Database user name'
         c.option 'password', '--password PW', "Database user's password, (default: '')"
         c.option 'host', '--host HOST', 'Database host name (default: "localhost")'
+        c.option 'port', '--port PORT', 'Custom database port connect to (optional)'
+        c.option 'blog_id', '--blog_id ID', 'Specify a single Movable Type blog ID to import (default: all blogs)'
+        c.option 'categories', '--categories', "If true, save post's categories in its YAML front matter. (default: true)"
+        c.option 'src_encoding', '--src_encoding ENCODING', "Encoding of strings from database. (default: UTF-8)"
+        c.option 'dest_encoding', '--dest_encoding ENCODING', "Encoding of output strings. (default: UTF-8)"
+        c.option 'comments','--comments', "If true, output comments in _comments directory (default: false)"
       end
 
       # By default this migrator will include posts for all your MovableType blogs.
@@ -58,6 +65,7 @@ module JekyllImport
       def self.process(options)
         options  = default_options.merge(options)
 
+        engine   = options.fetch('engine', 'mysql')
         dbname   = options.fetch('dbname')
         user     = options.fetch('user')
         pass     = options.fetch('password', "")
@@ -66,7 +74,19 @@ module JekyllImport
 
         posts_name_by_id = { } if comments
 
-        db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host)
+        db_opts = { user => user, :password => pass, :host => host }
+
+        # Use the default port by default, but set the port if one is provided.
+        if options['port']
+          db_opts[:port] = options['port']
+        end
+
+        if engine != 'mysql' && engine != 'postgres'
+          abort("unsupported --engine value supplied '#{engine}'. Only 'mysql' and 'postgres' are supported")
+        end 
+
+        db = Sequel.public_send(engine, dbname, db_opts)
+
         post_categories = db[:mt_placement].join(:mt_category, :category_id => :placement_category_id)
 
         FileUtils.mkdir_p "_posts"

data/lib/jekyll-import/importers/tumblr.rb

@@ -152,9 +152,11 @@ module JekyllImport
         title = "no title" if title.empty?
         slug = if post["slug"] && post["slug"].strip != ""
           post["slug"]
-        else
-          slug = title.downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '')
+        elsif title && title.downcase.gsub(/[^a-z0-9\-]/, '') != '' && title != 'no title'
+          slug = title.downcase.strip.gsub(' ', '-').gsub(/[^a-z0-9\-]/, '')
           slug.length > 200 ? slug.slice(0..200) : slug
+        else
+          slug = post['id']
         end
         {
           :name => "#{date}-#{slug}.#{format}",
@@ -205,7 +207,7 @@ module JekyllImport
           # Create an initial empty file for the post so that
           # we can instantiate a post object.
           File.open("_posts/tumblr/#{post[:name]}", "w")
-          tumblr_url = URI.parse(post[:slug]).path
+          tumblr_url = URI.parse(URI.encode(post[:slug])).path
           jekyll_url = Jekyll::Post.new(site, Dir.pwd, "", "tumblr/" + post[:name]).url
           redirect_dir = tumblr_url.sub(/\//, "") + "/"
           FileUtils.mkdir_p redirect_dir

data/lib/jekyll-import/importers/wordpressdotcom.rb

@@ -11,6 +11,7 @@ module JekyllImport
           hpricot
           time
           open-uri
+          open_uri_redirections
         ])
       end
 
@@ -38,14 +39,15 @@ module JekyllImport
             next
           end
           begin
-            open(uri) {|f|
+            open(uri, allow_redirections: :safe) {|f|
               File.open(dst, "wb") do |out|
                 out.puts f.read
               end
             }
             puts "    OK!"
           rescue => e
-            puts "    Errorr: #{e.message}"
+            puts "    Error: #{e.message}"
+            puts e.backtrace.join("\n")
           end
         end
       end

data/lib/jekyll-import/version.rb

@@ -1,3 +1,3 @@
 module JekyllImport
-  VERSION = '0.8.0'
+  VERSION = '0.9.0'
 end

data/lib/jekyll/commands/import.rb

@@ -14,6 +14,7 @@ module Jekyll
         :drupal7 => 'Drupal7',
         :enki => 'Enki',
         :joomla => 'Joomla',
+        :joomla3 => 'Joomla3',
         :jrnl => 'Jrnl',
         :ghost => 'Ghost',
         :google_reader => 'GoogleReader',

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-import
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Tom Preston-Werner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-13 00:00:00.000000000 Z
+date: 2015-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jekyll
@@ -276,6 +276,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: open_uri_redirections
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: launchy
   requirement: !ruby/object:Gem::Requirement
@@ -313,6 +327,7 @@ files:
 - lib/jekyll-import/importers/ghost.rb
 - lib/jekyll-import/importers/google_reader.rb
 - lib/jekyll-import/importers/joomla.rb
+- lib/jekyll-import/importers/joomla3.rb
 - lib/jekyll-import/importers/jrnl.rb
 - lib/jekyll-import/importers/marley.rb
 - lib/jekyll-import/importers/mephisto.rb
@@ -349,7 +364,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.3
+rubygems_version: 2.2.5
 signing_key: 
 specification_version: 2
 summary: Import command for Jekyll (static site generator).