caramelize 0.1.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +21 -53
- data/.travis.yml +5 -0
- data/CODE_OF_CONDUCT.md +76 -0
- data/Gemfile +3 -3
- data/LICENSE.md +1 -1
- data/README.md +48 -46
- data/Rakefile +8 -7
- data/bin/caramelize +73 -10
- data/caramelize.gemspec +28 -24
- data/lib/caramelize.rb +14 -0
- data/lib/caramelize/caramel.rb +48 -44
- data/lib/caramelize/content_transferer.rb +126 -69
- data/lib/caramelize/database_connector.rb +3 -6
- data/lib/caramelize/filter_processor.rb +27 -0
- data/lib/caramelize/filters/remove_table_tab_line_endings.rb +15 -0
- data/lib/caramelize/filters/swap_wiki_links.rb +26 -0
- data/lib/caramelize/filters/wikka_to_markdown.rb +67 -0
- data/lib/caramelize/health_check.rb +85 -0
- data/lib/caramelize/input_wiki/redmine_wiki.rb +120 -0
- data/lib/caramelize/input_wiki/wiki.rb +59 -0
- data/lib/caramelize/input_wiki/wikkawiki.rb +69 -0
- data/lib/caramelize/output_wiki/gollum.rb +80 -0
- data/lib/caramelize/page.rb +38 -14
- data/lib/caramelize/services/page_builder.rb +20 -0
- data/lib/caramelize/version.rb +1 -1
- data/spec/fixtures/markup/swap-links-input.textile +57 -0
- data/spec/fixtures/markup/swap-links-output.textile +57 -0
- data/spec/fixtures/markup/table-tab-line-endings-input.textile +145 -0
- data/spec/fixtures/markup/table-tab-line-endings-output.textile +145 -0
- data/spec/lib/caramelize/content_transferer_spec.rb +9 -0
- data/spec/lib/caramelize/filter_processor_spec.rb +34 -0
- data/spec/lib/caramelize/filters/remove_table_tab_line_endings_spec.rb +49 -0
- data/spec/lib/caramelize/filters/swap_wiki_links_spec.rb +49 -0
- data/spec/lib/caramelize/filters/wikka_to_markdown_spec.rb +198 -0
- data/spec/lib/caramelize/input_wiki/wiki_spec.rb +57 -0
- data/spec/lib/caramelize/output_wiki/gollum_spec.rb +113 -0
- data/spec/lib/caramelize/page_spec.rb +67 -0
- data/spec/lib/caramelize/services/page_builder.rb +29 -0
- data/spec/spec_helper.rb +8 -0
- metadata +165 -54
- data/lib/caramelize/author.rb +0 -8
- data/lib/caramelize/cli.rb +0 -80
- data/lib/caramelize/cli/create_command.rb +0 -52
- data/lib/caramelize/cli/run_command.rb +0 -33
- data/lib/caramelize/ext.rb +0 -17
- data/lib/caramelize/gollum_output.rb +0 -56
- data/lib/caramelize/wiki/redmine_wiki.rb +0 -60
- data/lib/caramelize/wiki/trac_converter.rb +0 -82
- data/lib/caramelize/wiki/wiki.rb +0 -41
- data/lib/caramelize/wiki/wikka_converter.rb +0 -38
- data/lib/caramelize/wiki/wikkawiki.rb +0 -55
- data/test/helper.rb +0 -18
- data/test/test_caramelize.rb +0 -7
data/caramelize.gemspec
CHANGED
@@ -1,29 +1,33 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
4
|
require "caramelize/version"
|
4
5
|
|
5
|
-
Gem::Specification.new do |
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
s.description = %q{By defining the connectors from the input wiki you can migrate any wiki to git-based Gollum wiki repositories.}
|
15
|
-
|
16
|
-
s.bindir = 'bin'
|
17
|
-
|
18
|
-
s.add_dependency('mysql2')
|
19
|
-
s.add_dependency('cmdparse')
|
20
|
-
s.add_dependency('ruby-progressbar')
|
21
|
-
s.add_dependency('gollum', '>= 1.3.0') # grit dependency implicit through gollum
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "caramelize"
|
8
|
+
spec.version = Caramelize::VERSION
|
9
|
+
spec.license = "MIT"
|
10
|
+
spec.authors = ["Daniel Senff"]
|
11
|
+
spec.email = ["mail@danielsenff.de"]
|
12
|
+
spec.homepage = "http://github.com/Dahie/caramelize"
|
13
|
+
spec.summary = %q{Flexible and modular wiki conversion tool}
|
14
|
+
spec.description = %q{With Caramelize you can migrate any wiki to git-based Gollum wiki repositories.}
|
22
15
|
|
23
|
-
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
24
20
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
21
|
+
spec.add_dependency('commander')
|
22
|
+
spec.add_dependency('gollum-lib')
|
23
|
+
spec.add_dependency('mysql2')
|
24
|
+
spec.add_dependency('ruby-progressbar')
|
25
|
+
|
26
|
+
spec.add_development_dependency "bundler", "~> 1.6"
|
27
|
+
spec.add_development_dependency "rake"
|
28
|
+
spec.add_development_dependency "rspec"
|
29
|
+
spec.add_development_dependency "byebug"
|
30
|
+
spec.add_development_dependency "rubocop"
|
31
|
+
spec.add_development_dependency "guard"
|
32
|
+
spec.add_development_dependency "guard-rspec"
|
29
33
|
end
|
data/lib/caramelize.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'caramelize/version'
|
2
|
+
require 'caramelize/page'
|
3
|
+
require 'caramelize/content_transferer'
|
4
|
+
require 'caramelize/filter_processor'
|
5
|
+
require 'caramelize/database_connector'
|
6
|
+
require 'caramelize/health_check'
|
7
|
+
require 'caramelize/output_wiki/gollum'
|
8
|
+
require 'caramelize/services/page_builder'
|
9
|
+
require 'caramelize/input_wiki/wiki'
|
10
|
+
require 'caramelize/input_wiki/redmine_wiki'
|
11
|
+
require 'caramelize/input_wiki/wikkawiki'
|
12
|
+
|
13
|
+
Encoding.default_external = Encoding::UTF_8
|
14
|
+
Encoding.default_internal = Encoding::UTF_8
|
data/lib/caramelize/caramel.rb
CHANGED
@@ -1,74 +1,78 @@
|
|
1
|
-
require 'caramelize/
|
2
|
-
require 'caramelize/
|
3
|
-
require 'caramelize/wiki/redmine_wiki'
|
1
|
+
require 'caramelize/input_wiki/wikkawiki'
|
2
|
+
require 'caramelize/input_wiki/redmine_wiki'
|
4
3
|
|
5
4
|
## Example caramelize configuration file
|
6
5
|
|
7
|
-
# Within this method you can define your own Wiki-Connectors to Wikis
|
6
|
+
# Within this method you can define your own Wiki-Connectors to Wikis
|
7
|
+
# not supported by default in this software
|
8
8
|
|
9
9
|
# Note, if you want to activate this, you need to uncomment the line below.
|
10
10
|
def customized_wiki
|
11
|
-
|
12
|
-
# This example is a reimplementation of the WikkaWiki-Connector.
|
11
|
+
|
12
|
+
# This example is a reimplementation of the WikkaWiki-Connector.
|
13
13
|
# To connect to WikkaWiki, I suggest to use the predefined Connector below.
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
14
|
+
options = { host: "localhost",
|
15
|
+
username: "user",
|
16
|
+
database: "database_name",
|
17
|
+
password: 'Picard-Delta-5',
|
18
|
+
markup: :wikka}
|
19
|
+
wiki = Caramelize::InputWiki::Wiki.new(options)
|
19
20
|
wiki.instance_eval do
|
20
21
|
def read_pages
|
21
22
|
sql = "SELECT id, tag, body, time, latest, user, note FROM wikka_pages ORDER BY time;"
|
22
|
-
@revisions, @titles = [], []
|
23
23
|
results = database.query(sql)
|
24
24
|
results.each do |row|
|
25
|
-
|
25
|
+
titles << row["tag"]
|
26
26
|
author = @authors[row["user"]]
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
27
|
+
properties = { id: row["id"],
|
28
|
+
title: row["tag"],
|
29
|
+
body: row["body"],
|
30
|
+
markup: 'wikka',
|
31
|
+
latest: row["latest"] == "Y",
|
32
|
+
time: row["time"],
|
33
|
+
message: row["note"],
|
34
|
+
author: author,
|
35
|
+
author_name: row["user"]}
|
36
|
+
page = Page.new(properties)
|
37
|
+
revisions << page
|
37
38
|
end
|
38
|
-
|
39
|
-
|
40
|
-
|
39
|
+
titles.uniq!
|
40
|
+
revisions
|
41
41
|
end
|
42
42
|
end
|
43
|
-
|
43
|
+
|
44
44
|
wiki
|
45
45
|
end
|
46
46
|
|
47
47
|
|
48
|
-
# if you want to use one of the preset Wiki-Connectors uncomment the connector
|
48
|
+
# if you want to use one of the preset Wiki-Connectors uncomment the connector
|
49
49
|
# and edit the database logins accordingly.
|
50
50
|
def predefined_wiki
|
51
|
-
|
51
|
+
|
52
52
|
# For connection to a WikkaWiki-Database use this Connector
|
53
|
-
#
|
54
|
-
# :
|
55
|
-
# :
|
56
|
-
# :
|
57
|
-
|
58
|
-
|
53
|
+
#options = { host: "localhost",
|
54
|
+
# username: "root",
|
55
|
+
# password: "root",
|
56
|
+
# database: "wikka" }
|
57
|
+
#return Caramelize::InputWiki::WikkaWiki.new(options)
|
58
|
+
|
59
|
+
|
59
60
|
# For connection to a Redmine-Database use this Connector
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
61
|
+
# Additional options:
|
62
|
+
# :create_namespace_overview => true/false (Default: true) - Creates a new wikipage at /home as root page for Gollum wiki
|
63
|
+
options = { host: "localhost",
|
64
|
+
username: "root",
|
65
|
+
password: "root",
|
66
|
+
database: "redmine_development" }
|
67
|
+
return Caramelize::InputWiki::RedmineWiki.new(options)
|
64
68
|
end
|
65
69
|
|
66
70
|
|
67
71
|
def input_wiki
|
68
|
-
|
69
|
-
#
|
72
|
+
# comment and uncomment to easily switch between predefined and
|
73
|
+
# costumized Wiki-connectors.
|
74
|
+
|
70
75
|
#return customized_wiki
|
71
|
-
|
72
|
-
return predefined_wiki
|
73
76
|
|
74
|
-
|
77
|
+
return predefined_wiki
|
78
|
+
end
|
@@ -1,79 +1,136 @@
|
|
1
|
-
#Encoding: UTF-8
|
2
|
-
|
3
|
-
require 'gollum'
|
4
|
-
require 'grit'
|
5
1
|
require 'ruby-progressbar'
|
6
2
|
|
7
3
|
module Caramelize
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
4
|
+
|
5
|
+
require 'caramelize/page'
|
6
|
+
require 'caramelize/content_transferer'
|
7
|
+
require 'caramelize/database_connector'
|
8
|
+
require 'caramelize/output_wiki/gollum'
|
9
|
+
require 'caramelize/input_wiki/redmine_wiki'
|
10
|
+
require 'caramelize/input_wiki/wikkawiki'
|
11
|
+
|
16
12
|
class ContentTransferer
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
13
|
+
attr_reader :input_wiki, :options
|
14
|
+
|
15
|
+
DEFAULT_GOLLUM_HOME_TITLE = 'Home'.freeze
|
16
|
+
|
17
|
+
def initialize(input_wiki, options)
|
18
|
+
@input_wiki = input_wiki
|
19
|
+
@options = options
|
20
|
+
|
21
|
+
options[:default_author] = options.fetch(:default_author, "Caramelize")
|
22
|
+
options[:markup] = target_markup
|
23
|
+
end
|
24
|
+
|
25
|
+
def execute
|
26
|
+
input_wiki.read_authors
|
27
|
+
|
28
|
+
commit_history
|
29
|
+
|
30
|
+
if verbose?
|
31
|
+
puts "From markup: #{input_wiki.markup.to_s}"
|
32
|
+
puts "To markup: #{target_markup.to_s}"
|
33
|
+
puts "Convert latest revisions:"
|
34
|
+
end
|
35
|
+
|
36
|
+
migrate_markup_of_latest_revisions
|
37
|
+
|
38
|
+
puts 'Create Namespace Overview' if verbose?
|
39
|
+
create_overview_page_of_namespaces if options[:create_namespace_overview]
|
40
|
+
|
41
|
+
rename_home_page
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def target_markup
|
47
|
+
@target_markup ||=
|
48
|
+
needs_conversion_to_target_markup? ? input_wiki.markup : :markdown
|
49
|
+
end
|
50
|
+
|
51
|
+
def needs_conversion_to_target_markup?
|
52
|
+
output_wiki.supported_markup.index(input_wiki.markup)
|
53
|
+
end
|
54
|
+
|
55
|
+
def revisions
|
56
|
+
@revisions ||= input_wiki.read_pages
|
57
|
+
end
|
58
|
+
|
59
|
+
def output_wiki
|
60
|
+
@output_wiki ||= OutputWiki::Gollum.new(options[:target])
|
61
|
+
end
|
62
|
+
|
63
|
+
def filter_processor
|
64
|
+
@filter_processor ||= FilterProcessor.new(input_wiki)
|
65
|
+
end
|
66
|
+
|
67
|
+
def verbose?
|
68
|
+
options[:verbose]
|
69
|
+
end
|
70
|
+
|
71
|
+
def revisions_count
|
72
|
+
revisions.count
|
73
|
+
end
|
74
|
+
|
75
|
+
def latest_revisions_count
|
76
|
+
input_wiki.latest_revisions.count
|
77
|
+
end
|
78
|
+
|
79
|
+
def create_overview_page_of_namespaces
|
80
|
+
output_wiki.commit_namespace_overview(input_wiki.namespaces)
|
81
|
+
end
|
82
|
+
|
83
|
+
def migrate_markup_progress_bar
|
84
|
+
@migrate_markup_progress_bar ||=
|
85
|
+
ProgressBar.create(title: "Markup filters",
|
86
|
+
total: latest_revisions_count)
|
87
|
+
end
|
88
|
+
|
89
|
+
def commit_history_progress_bar
|
90
|
+
@commit_history_progress_bar ||=
|
91
|
+
ProgressBar.create(title: "Revisions",
|
92
|
+
total: revisions_count)
|
93
|
+
end
|
94
|
+
|
95
|
+
def migrate_markup_of_latest_revisions
|
96
|
+
input_wiki.latest_revisions.each do |revision|
|
97
|
+
migrate_markup_of_revision(revision)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def commit_history
|
102
|
+
output_wiki.commit_history(revisions, options) do |page, index|
|
103
|
+
if verbose?
|
104
|
+
puts "(#{index + 1}/#{revisions_count}) #{page.time} #{page.title}"
|
40
105
|
else
|
41
|
-
|
106
|
+
commit_history_progress_bar.increment
|
42
107
|
end
|
43
108
|
end
|
109
|
+
end
|
44
110
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
body_new = original_wiki.to_textile rev.body
|
64
|
-
end
|
65
|
-
|
66
|
-
unless body_new.eql? rev.body
|
67
|
-
rev.body = body_new
|
68
|
-
rev.author_name = options[:markup]
|
69
|
-
rev.time = Time.now
|
70
|
-
rev.author = nil
|
71
|
-
|
72
|
-
# commit as latest page revision
|
73
|
-
output_wiki.commit_revision rev, options[:markup]
|
74
|
-
end
|
75
|
-
end
|
111
|
+
def migrate_markup_of_revision(revision)
|
112
|
+
if verbose?
|
113
|
+
puts "Filter source: #{revision.title} #{revision.time}"
|
114
|
+
else
|
115
|
+
migrate_markup_progress_bar.increment
|
116
|
+
end
|
117
|
+
|
118
|
+
body_new = filter_processor.run(revision.body)
|
119
|
+
|
120
|
+
unless body_new == revision.body
|
121
|
+
revision.body = body_new
|
122
|
+
revision.author_name = 'Caramelize'
|
123
|
+
revision.time = Time.now
|
124
|
+
revision.author = nil
|
125
|
+
revision.message = "Markup of '#{revision.title}' converted to #{target_markup}"
|
126
|
+
|
127
|
+
# commit as latest page revision
|
128
|
+
output_wiki.commit_revision(revision, options[:markup])
|
76
129
|
end
|
77
|
-
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def rename_home_page
|
133
|
+
output_wiki.rename_page(options[:home_page_title], DEFAULT_GOLLUM_HOME_TITLE)
|
134
|
+
end
|
78
135
|
end
|
79
|
-
end
|
136
|
+
end
|
@@ -2,19 +2,16 @@ require 'mysql2'
|
|
2
2
|
|
3
3
|
module Caramelize
|
4
4
|
module DatabaseConnector
|
5
|
-
|
5
|
+
|
6
6
|
def database
|
7
|
-
socket = ["/tmp/mysqld.sock",
|
8
|
-
"/tmp/mysql.sock",
|
7
|
+
socket = ["/tmp/mysqld.sock",
|
8
|
+
"/tmp/mysql.sock",
|
9
9
|
"/var/run/mysqld/mysqld.sock",
|
10
10
|
"/opt/local/var/run/mysql5/mysqld.sock",
|
11
11
|
"/var/lib/mysql/mysql.sock"].detect{|socket| File.exist?(socket) }
|
12
12
|
@options[:socket] = socket
|
13
13
|
@client = Mysql2::Client.new(@options) unless @client
|
14
14
|
@client
|
15
|
-
|
16
15
|
end
|
17
|
-
|
18
16
|
end
|
19
|
-
|
20
17
|
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Caramelize
|
2
|
+
class FilterProcessor
|
3
|
+
attr_reader :filters, :input_wiki
|
4
|
+
|
5
|
+
def initialize(input_wiki)
|
6
|
+
@filters = []
|
7
|
+
@input_wiki = input_wiki
|
8
|
+
|
9
|
+
initialize_wiki_filters
|
10
|
+
end
|
11
|
+
|
12
|
+
def run(body)
|
13
|
+
body_new = body
|
14
|
+
filters.each do |filter|
|
15
|
+
body_new = filter.new(body_new).run
|
16
|
+
end
|
17
|
+
body_new
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def initialize_wiki_filters
|
23
|
+
filters << input_wiki.filters
|
24
|
+
filters.flatten!
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|