bunto-import 2.0.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +21 -21
  3. data/README.markdown +33 -33
  4. data/lib/bunto-import.rb +49 -49
  5. data/lib/bunto-import/importer.rb +26 -26
  6. data/lib/bunto-import/importers.rb +10 -10
  7. data/lib/bunto-import/importers/behance.rb +80 -80
  8. data/lib/bunto-import/importers/blogger.rb +330 -264
  9. data/lib/bunto-import/importers/csv.rb +96 -96
  10. data/lib/bunto-import/importers/drupal6.rb +53 -139
  11. data/lib/bunto-import/importers/drupal7.rb +54 -111
  12. data/lib/bunto-import/importers/drupal_common.rb +157 -0
  13. data/lib/bunto-import/importers/easyblog.rb +96 -96
  14. data/lib/bunto-import/importers/enki.rb +74 -74
  15. data/lib/bunto-import/importers/ghost.rb +68 -68
  16. data/lib/bunto-import/importers/google_reader.rb +64 -64
  17. data/lib/bunto-import/importers/joomla.rb +92 -90
  18. data/lib/bunto-import/importers/joomla3.rb +91 -91
  19. data/lib/bunto-import/importers/jrnl.rb +125 -125
  20. data/lib/bunto-import/importers/marley.rb +72 -72
  21. data/lib/bunto-import/importers/mephisto.rb +99 -99
  22. data/lib/bunto-import/importers/mt.rb +257 -257
  23. data/lib/bunto-import/importers/posterous.rb +130 -130
  24. data/lib/bunto-import/importers/rss.rb +62 -62
  25. data/lib/bunto-import/importers/s9y.rb +60 -60
  26. data/lib/bunto-import/importers/s9y_database.rb +363 -0
  27. data/lib/bunto-import/importers/textpattern.rb +70 -70
  28. data/lib/bunto-import/importers/tumblr.rb +300 -289
  29. data/lib/bunto-import/importers/typo.rb +88 -88
  30. data/lib/bunto-import/importers/wordpress.rb +372 -372
  31. data/lib/bunto-import/importers/wordpressdotcom.rb +207 -207
  32. data/lib/bunto-import/util.rb +76 -76
  33. data/lib/bunto-import/version.rb +3 -3
  34. data/lib/bunto/commands/import.rb +79 -79
  35. metadata +84 -54
@@ -0,0 +1,157 @@
1
+ require 'date'
2
+
3
+ module BuntoImport
4
+ module Importers
5
+ module DrupalCommon
6
+ # This module provides a base for the Drupal importers (at least for 6
7
+ # and 7; since 8 will be a different beast). Version-specific importers
8
+ # will need to implement the missing methods from the Importer class.
9
+ #
10
+ # The general idea is that this importer reads a MySQL database via Sequel
11
+ # and creates a post file for each node it finds in the Drupal database.
12
+
13
+ module ClassMethods
14
+ DEFAULTS = {
15
+ "password" => "",
16
+ "host" => "localhost",
17
+ "prefix" => "",
18
+ "types" => %w(blog story article)
19
+ }
20
+
21
+ def specify_options(c)
22
+ c.option 'dbname', '--dbname DB', 'Database name'
23
+ c.option 'user', '--user USER', 'Database user name'
24
+ c.option 'password', '--password PW', "Database user's password (default: #{DEFAULTS["password"].inspect})"
25
+ c.option 'host', '--host HOST', "Database host name (default: #{DEFAULTS["host"].inspect})"
26
+ c.option 'prefix', '--prefix PREFIX', "Table prefix name (default: #{DEFAULTS["prefix"].inspect})"
27
+ c.option 'types', '--types TYPE1[,TYPE2[,TYPE3...]]', Array,
28
+ "The Drupal content types to be imported (default: #{DEFAULTS["types"].join(",")})"
29
+ end
30
+
31
+ def require_deps
32
+ BuntoImport.require_with_fallback(%w[
33
+ rubygems
34
+ sequel
35
+ fileutils
36
+ safe_yaml
37
+ ])
38
+ end
39
+
40
+ def process(options)
41
+ dbname = options.fetch('dbname')
42
+ user = options.fetch('user')
43
+ pass = options.fetch('password', DEFAULTS["password"])
44
+ host = options.fetch('host', DEFAULTS["host"])
45
+ prefix = options.fetch('prefix', DEFAULTS["prefix"])
46
+ types = options.fetch('types', DEFAULTS["types"])
47
+
48
+ db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
49
+
50
+ query = self.build_query(prefix, types)
51
+
52
+ conf = Bunto.configuration({})
53
+ src_dir = conf['source']
54
+
55
+ dirs = {
56
+ :_posts => File.join(src_dir, '_posts').to_s,
57
+ :_drafts => File.join(src_dir, '_drafts').to_s,
58
+ :_layouts => Bunto.sanitized_path(src_dir, conf['layouts_dir'].to_s)
59
+ }
60
+
61
+ dirs.each do |key, dir|
62
+ FileUtils.mkdir_p dir
63
+ end
64
+
65
+ # Create the refresh layout
66
+ # Change the refresh url if you customized your permalink config
67
+ File.open(File.join(dirs[:_layouts], 'refresh.html'), 'w') do |f|
68
+ f.puts <<-HTML
69
+ <!DOCTYPE html>
70
+ <html>
71
+ <head>
72
+ <meta http-equiv="content-type" content="text/html; charset=utf-8" />
73
+ <meta http-equiv="refresh" content="0;url={{ page.refresh_to_post_id }}.html" />
74
+ </head>
75
+ </html>
76
+ HTML
77
+ end
78
+
79
+ db[query].each do |post|
80
+ # Get required fields
81
+ data, content = self.post_data(post)
82
+
83
+ data['layout'] = post[:type]
84
+ title = data['title'] = post[:title].strip.force_encoding('UTF-8')
85
+ time = data['created'] = post[:created]
86
+
87
+ # Get the relevant fields as a hash and delete empty fields
88
+ data = data.delete_if { |k,v| v.nil? || v == ''}.each_pair {
89
+ |k,v| ((v.is_a? String) ? v.force_encoding('UTF-8') : v)
90
+ }
91
+
92
+ # Construct a Bunto compatible file name
93
+ is_published = post[:status] == 1
94
+ node_id = post[:nid]
95
+ dir = is_published ? dirs[:_posts] : dirs[:_drafts]
96
+ slug = title.strip.downcase.gsub(/(&|&amp;)/, ' and ').gsub(/[\s\.\/\\]/, '-').gsub(/[^\w-]/, '').gsub(/[-_]{2,}/, '-').gsub(/^[-_]/, '').gsub(/[-_]$/, '')
97
+ filename = Time.at(time).to_datetime.strftime('%Y-%m-%d-') + slug + '.md'
98
+
99
+ # Write out the data and content to file
100
+ File.open("#{dir}/#{filename}", 'w') do |f|
101
+ f.puts data.to_yaml
102
+ f.puts '---'
103
+ f.puts content
104
+ end
105
+
106
+
107
+ # Make a file to redirect from the old Drupal URL
108
+ if is_published
109
+ alias_query = self.aliases_query(prefix)
110
+ type = post[:type]
111
+
112
+ aliases = db[alias_query, "#{type}/#{node_id}"].all
113
+
114
+ aliases.push(:alias => "#{type}/#{node_id}")
115
+
116
+ aliases.each do |url_alias|
117
+ FileUtils.mkdir_p url_alias[:alias]
118
+ File.open("#{url_alias[:alias]}/index.md", "w") do |f|
119
+ f.puts '---'
120
+ f.puts 'layout: refresh'
121
+ f.puts "refresh_to_post_id: /#{Time.at(time).to_datetime.strftime('%Y/%m/%d/') + slug}"
122
+ f.puts '---'
123
+ end
124
+ end
125
+ end
126
+ end
127
+ end
128
+ end
129
+
130
+ def build_query(prefix, types)
131
+ raise 'The importer you are trying to use does not implement the get_query() method.'
132
+ end
133
+
134
+ def aliases_query(prefix)
135
+ # Make sure you implement the query returning "alias" as the column name
136
+ # for the URL aliases. See the Drupal 6 importer for an example. The
137
+ # alias field is called 'dst' but we alias it to 'alias', to follow
138
+ # Drupal 7's column names.
139
+ raise 'The importer you are trying to use does not implement the get_aliases_query() method.'
140
+ end
141
+
142
+ def post_data(sql_post_data)
143
+ raise 'The importer you are trying to use does not implement the get_query() method.'
144
+ end
145
+
146
+ def validate(options)
147
+ %w[dbname user].each do |option|
148
+ if options[option].nil?
149
+ abort "Missing mandatory option --#{option}."
150
+ end
151
+ end
152
+ end
153
+
154
+
155
+ end
156
+ end
157
+ end
@@ -1,96 +1,96 @@
1
- module BuntoImport
2
- module Importers
3
- class Easyblog < Importer
4
- def self.validate(options)
5
- %w[dbname user].each do |option|
6
- if options[option].nil?
7
- abort "Missing mandatory option --#{option}."
8
- end
9
- end
10
- end
11
-
12
- def self.specify_options(c)
13
- c.option 'dbname', '--dbname', 'Database name'
14
- c.option 'user', '--user', 'Database user name'
15
- c.option 'password', '--password', "Database user's password (default: '')"
16
- c.option 'host', '--host', 'Database host name'
17
- c.option 'section', '--section', 'Table prefix name'
18
- c.option 'prefix', '--prefix', 'Table prefix name'
19
- end
20
-
21
- def self.require_deps
22
- BuntoImport.require_with_fallback(%w[
23
- rubygems
24
- sequel
25
- fileutils
26
- safe_yaml
27
- ])
28
- end
29
-
30
- def self.process(options)
31
- dbname = options.fetch('dbname')
32
- user = options.fetch('user')
33
- pass = options.fetch('password', '')
34
- host = options.fetch('host', "localhost")
35
- section = options.fetch('section', '1')
36
- table_prefix = options.fetch('prefix', "jos_")
37
-
38
- db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
39
-
40
- FileUtils.mkdir_p("_posts")
41
-
42
- # Reads a MySQL database via Sequel and creates a post file for each
43
- # post in wp_posts that has post_status = 'publish'. This restriction is
44
- # made because 'draft' posts are not guaranteed to have valid dates.
45
-
46
- query = "
47
- select
48
- ep.`title`, `permalink` as alias, concat(`intro`, `content`) as content, ep.`created`, ep.`id`, ec.`title` as category, tags
49
- from
50
- #{table_prefix}easyblog_post ep
51
- left join #{table_prefix}easyblog_category ec on (ep.category_id = ec.id)
52
- left join (
53
- select
54
- ept.post_id,
55
- group_concat(et.alias order by alias separator ' ') as tags
56
- from
57
- #{table_prefix}easyblog_post_tag ept
58
- join #{table_prefix}easyblog_tag et on (ept.tag_id = et.id)
59
- group by
60
- ept.post_id) x on (ep.id = x.post_id);
61
- "
62
-
63
- db[query].each do |post|
64
- # Get required fields and construct Bunto compatible name.
65
- title = post[:title]
66
- slug = post[:alias]
67
- date = post[:created]
68
- content = post[:content]
69
- category = post[:category]
70
- tags = post[:tags]
71
- name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day,
72
- slug]
73
-
74
- # Get the relevant fields as a hash, delete empty fields and convert
75
- # to YAML for the header.
76
- data = {
77
- 'layout' => 'post',
78
- 'title' => title.to_s,
79
- 'joomla_id' => post[:id],
80
- 'joomla_url' => post[:alias],
81
- 'category' => post[:category],
82
- 'tags' => post[:tags],
83
- 'date' => date
84
- }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
85
-
86
- # Write out the data and content to file
87
- File.open("_posts/#{name}", "w") do |f|
88
- f.puts data
89
- f.puts "---"
90
- f.puts content
91
- end
92
- end
93
- end
94
- end
95
- end
96
- end
1
+ module BuntoImport
2
+ module Importers
3
+ class Easyblog < Importer
4
+ def self.validate(options)
5
+ %w[dbname user].each do |option|
6
+ if options[option].nil?
7
+ abort "Missing mandatory option --#{option}."
8
+ end
9
+ end
10
+ end
11
+
12
+ def self.specify_options(c)
13
+ c.option 'dbname', '--dbname', 'Database name'
14
+ c.option 'user', '--user', 'Database user name'
15
+ c.option 'password', '--password', "Database user's password (default: '')"
16
+ c.option 'host', '--host', 'Database host name'
17
+ c.option 'section', '--section', 'Table prefix name'
18
+ c.option 'prefix', '--prefix', 'Table prefix name'
19
+ end
20
+
21
+ def self.require_deps
22
+ BuntoImport.require_with_fallback(%w[
23
+ rubygems
24
+ sequel
25
+ fileutils
26
+ safe_yaml
27
+ ])
28
+ end
29
+
30
+ def self.process(options)
31
+ dbname = options.fetch('dbname')
32
+ user = options.fetch('user')
33
+ pass = options.fetch('password', '')
34
+ host = options.fetch('host', "localhost")
35
+ section = options.fetch('section', '1')
36
+ table_prefix = options.fetch('prefix', "jos_")
37
+
38
+ db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8')
39
+
40
+ FileUtils.mkdir_p("_posts")
41
+
42
+ # Reads a MySQL database via Sequel and creates a post file for each
43
+ # post in wp_posts that has post_status = 'publish'. This restriction is
44
+ # made because 'draft' posts are not guaranteed to have valid dates.
45
+
46
+ query = "
47
+ select
48
+ ep.`title`, `permalink` as alias, concat(`intro`, `content`) as content, ep.`created`, ep.`id`, ec.`title` as category, tags
49
+ from
50
+ #{table_prefix}easyblog_post ep
51
+ left join #{table_prefix}easyblog_category ec on (ep.category_id = ec.id)
52
+ left join (
53
+ select
54
+ ept.post_id,
55
+ group_concat(et.alias order by alias separator ' ') as tags
56
+ from
57
+ #{table_prefix}easyblog_post_tag ept
58
+ join #{table_prefix}easyblog_tag et on (ept.tag_id = et.id)
59
+ group by
60
+ ept.post_id) x on (ep.id = x.post_id);
61
+ "
62
+
63
+ db[query].each do |post|
64
+ # Get required fields and construct Bunto compatible name.
65
+ title = post[:title]
66
+ slug = post[:alias]
67
+ date = post[:created]
68
+ content = post[:content]
69
+ category = post[:category]
70
+ tags = post[:tags]
71
+ name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day,
72
+ slug]
73
+
74
+ # Get the relevant fields as a hash, delete empty fields and convert
75
+ # to YAML for the header.
76
+ data = {
77
+ 'layout' => 'post',
78
+ 'title' => title.to_s,
79
+ 'joomla_id' => post[:id],
80
+ 'joomla_url' => post[:alias],
81
+ 'category' => post[:category],
82
+ 'tags' => post[:tags],
83
+ 'date' => date
84
+ }.delete_if { |k,v| v.nil? || v == '' }.to_yaml
85
+
86
+ # Write out the data and content to file
87
+ File.open("_posts/#{name}", "w") do |f|
88
+ f.puts data
89
+ f.puts "---"
90
+ f.puts content
91
+ end
92
+ end
93
+ end
94
+ end
95
+ end
96
+ end
@@ -1,74 +1,74 @@
1
- module BuntoImport
2
- module Importers
3
- class Enki < Importer
4
- SQL = <<-EOS
5
- SELECT p.id,
6
- p.title,
7
- p.slug,
8
- p.body,
9
- p.published_at as date,
10
- p.cached_tag_list as tags
11
- FROM posts p
12
- EOS
13
-
14
- def self.validate(options)
15
- %w[dbname user].each do |option|
16
- if options[option].nil?
17
- abort "Missing mandatory option --#{option}."
18
- end
19
- end
20
- end
21
-
22
- def self.specify_options(c)
23
- c.option 'dbname', '--dbname', 'Database name'
24
- c.option 'user', '--user', 'Database name'
25
- c.option 'password', '--password', 'Database name (default: "")'
26
- c.option 'host', '--host', 'Database name'
27
- end
28
-
29
- def self.require_deps
30
- BuntoImport.require_with_fallback(%w[
31
- rubygems
32
- sequel
33
- fileutils
34
- pg
35
- yaml
36
- ])
37
- end
38
-
39
- # Just working with postgres, but can be easily adapted
40
- # to work with both mysql and postgres.
41
- def self.process(options)
42
- dbname = options.fetch('dbname')
43
- user = options.fetch('user')
44
- pass = options.fetch('password', "")
45
- host = options.fetch('host', "localhost")
46
-
47
- FileUtils.mkdir_p('_posts')
48
- db = Sequel.postgres(:database => dbname,
49
- :user => user,
50
- :password => pass,
51
- :host => host,
52
- :encoding => 'utf8')
53
-
54
- db[SQL].each do |post|
55
- name = [ sprintf("%.04d", post[:date].year),
56
- sprintf("%.02d", post[:date].month),
57
- sprintf("%.02d", post[:date].day),
58
- post[:slug].strip ].join('-')
59
- name += '.textile'
60
-
61
- File.open("_posts/#{name}", 'w') do |f|
62
- f.puts({ 'layout' => 'post',
63
- 'title' => post[:title].to_s,
64
- 'enki_id' => post[:id],
65
- 'categories' => post[:tags]
66
- }.delete_if { |k, v| v.nil? || v == '' }.to_yaml)
67
- f.puts '---'
68
- f.puts post[:body].delete("\r")
69
- end
70
- end
71
- end
72
- end
73
- end
74
- end
1
+ module BuntoImport
2
+ module Importers
3
+ class Enki < Importer
4
+ SQL = <<-EOS
5
+ SELECT p.id,
6
+ p.title,
7
+ p.slug,
8
+ p.body,
9
+ p.published_at as date,
10
+ p.cached_tag_list as tags
11
+ FROM posts p
12
+ EOS
13
+
14
+ def self.validate(options)
15
+ %w[dbname user].each do |option|
16
+ if options[option].nil?
17
+ abort "Missing mandatory option --#{option}."
18
+ end
19
+ end
20
+ end
21
+
22
+ def self.specify_options(c)
23
+ c.option 'dbname', '--dbname', 'Database name'
24
+ c.option 'user', '--user', 'Database name'
25
+ c.option 'password', '--password', 'Database name (default: "")'
26
+ c.option 'host', '--host', 'Database name'
27
+ end
28
+
29
+ def self.require_deps
30
+ BuntoImport.require_with_fallback(%w[
31
+ rubygems
32
+ sequel
33
+ fileutils
34
+ pg
35
+ yaml
36
+ ])
37
+ end
38
+
39
+ # Just working with postgres, but can be easily adapted
40
+ # to work with both mysql and postgres.
41
+ def self.process(options)
42
+ dbname = options.fetch('dbname')
43
+ user = options.fetch('user')
44
+ pass = options.fetch('password', "")
45
+ host = options.fetch('host', "localhost")
46
+
47
+ FileUtils.mkdir_p('_posts')
48
+ db = Sequel.postgres(:database => dbname,
49
+ :user => user,
50
+ :password => pass,
51
+ :host => host,
52
+ :encoding => 'utf8')
53
+
54
+ db[SQL].each do |post|
55
+ name = [ sprintf("%.04d", post[:date].year),
56
+ sprintf("%.02d", post[:date].month),
57
+ sprintf("%.02d", post[:date].day),
58
+ post[:slug].strip ].join('-')
59
+ name += '.textile'
60
+
61
+ File.open("_posts/#{name}", 'w') do |f|
62
+ f.puts({ 'layout' => 'post',
63
+ 'title' => post[:title].to_s,
64
+ 'enki_id' => post[:id],
65
+ 'categories' => post[:tags]
66
+ }.delete_if { |k, v| v.nil? || v == '' }.to_yaml)
67
+ f.puts '---'
68
+ f.puts post[:body].delete("\r")
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end