caramelize 0.1.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +21 -53
- data/.travis.yml +5 -0
- data/CODE_OF_CONDUCT.md +76 -0
- data/Gemfile +3 -3
- data/LICENSE.md +1 -1
- data/README.md +48 -46
- data/Rakefile +8 -7
- data/bin/caramelize +73 -10
- data/caramelize.gemspec +28 -24
- data/lib/caramelize.rb +14 -0
- data/lib/caramelize/caramel.rb +48 -44
- data/lib/caramelize/content_transferer.rb +126 -69
- data/lib/caramelize/database_connector.rb +3 -6
- data/lib/caramelize/filter_processor.rb +27 -0
- data/lib/caramelize/filters/remove_table_tab_line_endings.rb +15 -0
- data/lib/caramelize/filters/swap_wiki_links.rb +26 -0
- data/lib/caramelize/filters/wikka_to_markdown.rb +67 -0
- data/lib/caramelize/health_check.rb +85 -0
- data/lib/caramelize/input_wiki/redmine_wiki.rb +120 -0
- data/lib/caramelize/input_wiki/wiki.rb +59 -0
- data/lib/caramelize/input_wiki/wikkawiki.rb +69 -0
- data/lib/caramelize/output_wiki/gollum.rb +80 -0
- data/lib/caramelize/page.rb +38 -14
- data/lib/caramelize/services/page_builder.rb +20 -0
- data/lib/caramelize/version.rb +1 -1
- data/spec/fixtures/markup/swap-links-input.textile +57 -0
- data/spec/fixtures/markup/swap-links-output.textile +57 -0
- data/spec/fixtures/markup/table-tab-line-endings-input.textile +145 -0
- data/spec/fixtures/markup/table-tab-line-endings-output.textile +145 -0
- data/spec/lib/caramelize/content_transferer_spec.rb +9 -0
- data/spec/lib/caramelize/filter_processor_spec.rb +34 -0
- data/spec/lib/caramelize/filters/remove_table_tab_line_endings_spec.rb +49 -0
- data/spec/lib/caramelize/filters/swap_wiki_links_spec.rb +49 -0
- data/spec/lib/caramelize/filters/wikka_to_markdown_spec.rb +198 -0
- data/spec/lib/caramelize/input_wiki/wiki_spec.rb +57 -0
- data/spec/lib/caramelize/output_wiki/gollum_spec.rb +113 -0
- data/spec/lib/caramelize/page_spec.rb +67 -0
- data/spec/lib/caramelize/services/page_builder.rb +29 -0
- data/spec/spec_helper.rb +8 -0
- metadata +165 -54
- data/lib/caramelize/author.rb +0 -8
- data/lib/caramelize/cli.rb +0 -80
- data/lib/caramelize/cli/create_command.rb +0 -52
- data/lib/caramelize/cli/run_command.rb +0 -33
- data/lib/caramelize/ext.rb +0 -17
- data/lib/caramelize/gollum_output.rb +0 -56
- data/lib/caramelize/wiki/redmine_wiki.rb +0 -60
- data/lib/caramelize/wiki/trac_converter.rb +0 -82
- data/lib/caramelize/wiki/wiki.rb +0 -41
- data/lib/caramelize/wiki/wikka_converter.rb +0 -38
- data/lib/caramelize/wiki/wikkawiki.rb +0 -55
- data/test/helper.rb +0 -18
- data/test/test_caramelize.rb +0 -7
@@ -0,0 +1,15 @@
|
|
1
|
+
module Caramelize
|
2
|
+
class RemoveTableTabLineEndings
|
3
|
+
|
4
|
+
def initialize(body)
|
5
|
+
@body = body
|
6
|
+
end
|
7
|
+
|
8
|
+
# take an input stream and convert all wikka syntax to markdown syntax
|
9
|
+
def run
|
10
|
+
migrated_body = @body.dup
|
11
|
+
migrated_body.gsub!(/\|[\t ]*\r?[\n]/, "|\n")
|
12
|
+
migrated_body
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Caramelize
|
2
|
+
class SwapWikiLinks
|
3
|
+
def initialize(body)
|
4
|
+
@body = body
|
5
|
+
end
|
6
|
+
|
7
|
+
# take an input stream and convert all wikka syntax to markdown syntax
|
8
|
+
def run
|
9
|
+
migrated_body = @body.dup
|
10
|
+
|
11
|
+
migrated_body.gsub!(/\[\[(\S+)\|(.+?)\]\]/) { format_link($2, $1) }
|
12
|
+
migrated_body.gsub!(/\[\[([\w\s\-\.]*)\]\]/) { format_link($1, $1.dup) }
|
13
|
+
|
14
|
+
migrated_body
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def format_link(label, link)
|
20
|
+
link.downcase!
|
21
|
+
link.gsub!(' ', '_')
|
22
|
+
link.gsub!(/\./, '')
|
23
|
+
"[[#{label}|#{link}]]"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module Caramelize
|
2
|
+
class Wikka2Markdown
|
3
|
+
attr_reader :source_body
|
4
|
+
|
5
|
+
def initialize(source_body)
|
6
|
+
@source_body = source_body
|
7
|
+
end
|
8
|
+
|
9
|
+
def run
|
10
|
+
# TODO images: ({{image)(url\=?)?(.*)(}})
|
11
|
+
# markdown: 
|
12
|
+
|
13
|
+
replace_headlines
|
14
|
+
replace_formatting
|
15
|
+
replace_lists
|
16
|
+
replace_wiki_links
|
17
|
+
replace_links
|
18
|
+
replace_code_block
|
19
|
+
|
20
|
+
target_body
|
21
|
+
end
|
22
|
+
|
23
|
+
def replace_headlines
|
24
|
+
target_body.gsub!(/(======)(.*?)(======)/ ) {|s| '# ' + $2 } #h1
|
25
|
+
target_body.gsub!(/(=====)(.*?)(=====)/) {|s| '## ' + $2 } #h2
|
26
|
+
target_body.gsub!(/(====)(.*?)(====)/) {|s| '### ' + $2 } #h3
|
27
|
+
target_body.gsub!(/(===)(.*?)(===)/) {|s| '#### ' + $2 } #h4
|
28
|
+
target_body.gsub!(/(==)(.*?)(==)/) {|s| '##### ' + $2 } #h5
|
29
|
+
end
|
30
|
+
|
31
|
+
def replace_formatting
|
32
|
+
target_body.gsub!(/(\*\*)(.*?)(\*\*)/) {|s| '**' + $2 + '**' } #bold
|
33
|
+
target_body.gsub!(/(\/\/)(.*?)(\/\/)/, '*\2*') #italic
|
34
|
+
target_body.gsub!(/(__)(.*?)(__)/) {|s| '<u>' + $2 + '</u>'} #underline
|
35
|
+
target_body.gsub!(/(---)/, ' ') #forced linebreak
|
36
|
+
end
|
37
|
+
|
38
|
+
def replace_lists
|
39
|
+
target_body.gsub!(/(\t-\s?)(.*)/, '* \2') # unordered list
|
40
|
+
target_body.gsub!(/(~-\s?)(.*)/, '* \2') # unordered list
|
41
|
+
target_body.gsub!(/( -\s?)(.*)/, '* \2') # unordered list
|
42
|
+
|
43
|
+
target_body.gsub!(/(~1\)\s?)(.*)/, '1. \2') # unordered list
|
44
|
+
# TODO ordered lists
|
45
|
+
end
|
46
|
+
|
47
|
+
def replace_wiki_links
|
48
|
+
target_body.gsub!(/[\[]{2}(\w+)[\s|](.+?)[\]]{2}/, '[[\2|\1]]')
|
49
|
+
end
|
50
|
+
|
51
|
+
def replace_links
|
52
|
+
target_body.gsub!(/[\[]{2}((\w+):[\S][^\| ]*)[\| ](.*)[\]]{2}/,
|
53
|
+
'[\3](\1)')
|
54
|
+
target_body.gsub!(/[\[]{2}((\w+):.*)[\]]{2}/, '<\1>')
|
55
|
+
end
|
56
|
+
|
57
|
+
def replace_code_block
|
58
|
+
target_body.gsub!(/^%%\s(.*?)%%\s?/m) do
|
59
|
+
$1.gsub(/^/, ' ')
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def target_body
|
64
|
+
@target_body ||= source_body.dup
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
module Caramelize
|
2
|
+
class HealthCheck
|
3
|
+
attr_reader :wiki_path, :options
|
4
|
+
|
5
|
+
DEFAULT_GOLLUM_HOME_TITLE = 'Home'.freeze
|
6
|
+
|
7
|
+
def initialize(wiki_path, options={})
|
8
|
+
@wiki_path = wiki_path
|
9
|
+
@options = options
|
10
|
+
end
|
11
|
+
|
12
|
+
def execute
|
13
|
+
#puts page_paths.sort.inspect
|
14
|
+
|
15
|
+
check_pages
|
16
|
+
|
17
|
+
#puts intra_wiki_paths.sort.inspect
|
18
|
+
|
19
|
+
puts "\n # Pages not linked within Wiki:"
|
20
|
+
puts page_paths_without_intra_wiki_path.sort.inspect
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def files
|
26
|
+
@files ||= Dir.glob([wiki_path, '**/*.md'].join('/'))
|
27
|
+
end
|
28
|
+
|
29
|
+
def file_names
|
30
|
+
files.map do |file|
|
31
|
+
file.gsub("#{wiki_path}/", '').split('.').first
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def check_pages
|
36
|
+
pages.each do |page|
|
37
|
+
puts "\n## #{page.path}"
|
38
|
+
check_page(page)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def check_page(page)
|
43
|
+
intra_wiki_links = intra_wiki_links(page.text_data)
|
44
|
+
available = 0
|
45
|
+
intra_wiki_links.each do |link|
|
46
|
+
intra_wiki_link = page.path.split('/').first == page.path ? link : [page.path.split('/').first, link].join('/')
|
47
|
+
if !page_paths.include?(intra_wiki_link)
|
48
|
+
puts "#{intra_wiki_link} expected, but missing"
|
49
|
+
else
|
50
|
+
available += 1
|
51
|
+
intra_wiki_paths << intra_wiki_link
|
52
|
+
end
|
53
|
+
end
|
54
|
+
puts "#{available}/#{intra_wiki_links.count} available"
|
55
|
+
end
|
56
|
+
|
57
|
+
def intra_wiki_links(body)
|
58
|
+
body.scan(/\[\[(.+\|)?(\S+)\]\]/).map { |match| match[1] }.uniq
|
59
|
+
end
|
60
|
+
|
61
|
+
def pages
|
62
|
+
gollum.pages
|
63
|
+
end
|
64
|
+
|
65
|
+
def page_paths
|
66
|
+
pages.map(&:path).map { |path| path.split('.').first }
|
67
|
+
end
|
68
|
+
|
69
|
+
def intra_wiki_paths
|
70
|
+
@intra_wiki_paths ||= []
|
71
|
+
end
|
72
|
+
|
73
|
+
def page_paths_without_intra_wiki_path
|
74
|
+
page_paths - intra_wiki_paths
|
75
|
+
end
|
76
|
+
|
77
|
+
def check_home_page
|
78
|
+
puts "Home.md missing" if File.exist?('wiki-export/Home.md')
|
79
|
+
end
|
80
|
+
|
81
|
+
def gollum
|
82
|
+
@gollum ||= ::Gollum::Wiki.new(wiki_path)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
require 'caramelize/input_wiki/wiki'
|
2
|
+
require 'caramelize/filters/swap_wiki_links'
|
3
|
+
require 'caramelize/filters/remove_table_tab_line_endings'
|
4
|
+
|
5
|
+
module Caramelize
|
6
|
+
module InputWiki
|
7
|
+
class RedmineWiki < Wiki
|
8
|
+
include DatabaseConnector
|
9
|
+
|
10
|
+
def initialize(options = {})
|
11
|
+
super(options)
|
12
|
+
@options[:markup] = :textile
|
13
|
+
@options[:filters] << Caramelize::SwapWikiLinks
|
14
|
+
@options[:filters] << Caramelize::RemoveTableTabLineEndings
|
15
|
+
@options[:create_namespace_overview] = true
|
16
|
+
end
|
17
|
+
|
18
|
+
# after calling this action, I expect the titles and revisions to be filled
|
19
|
+
def read_pages
|
20
|
+
add_projects_as_namespaces
|
21
|
+
|
22
|
+
pages.each do |row_page|
|
23
|
+
build_page(row_page)
|
24
|
+
end
|
25
|
+
titles.uniq!
|
26
|
+
revisions.sort! { |a,b| a.time <=> b.time }
|
27
|
+
|
28
|
+
revisions
|
29
|
+
end
|
30
|
+
|
31
|
+
def read_authors
|
32
|
+
results = database.query(authors_query)
|
33
|
+
results.each do |row|
|
34
|
+
authors[row["id"]] = OpenStruct.new(id: row["id"],
|
35
|
+
name: row["login"],
|
36
|
+
email: row["mail"])
|
37
|
+
end
|
38
|
+
authors
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def build_page(row_page)
|
44
|
+
results_contents = database.query(single_page_query(row_page['id']))
|
45
|
+
|
46
|
+
wiki = wikis.select{ |row| row['id'] == row_page['wiki_id'] }.first
|
47
|
+
|
48
|
+
project_identifier = ''
|
49
|
+
|
50
|
+
if wiki
|
51
|
+
project = projects.select{ |row| row['id'] == wiki['project_id'] }.first
|
52
|
+
project_identifier = project['identifier'] + '/'
|
53
|
+
end
|
54
|
+
|
55
|
+
title = project_identifier + row_page['title']
|
56
|
+
titles << title
|
57
|
+
|
58
|
+
results_contents.each do |row_content|
|
59
|
+
page = Page.new(build_properties(title, row_content))
|
60
|
+
revisions << page
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def add_projects_as_namespaces
|
65
|
+
projects.each do |row_project|
|
66
|
+
namespace = OpenStruct.new(identifier: row_project['identifier'],
|
67
|
+
name: row_project['name'])
|
68
|
+
namespaces << namespace
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def authors_query
|
73
|
+
'SELECT id, login, mail FROM users;'
|
74
|
+
end
|
75
|
+
|
76
|
+
def single_page_query(page_id)
|
77
|
+
"SELECT * FROM wiki_content_versions WHERE page_id='#{page_id}' ORDER BY updated_on;"
|
78
|
+
end
|
79
|
+
|
80
|
+
def projects_query
|
81
|
+
'SELECT id, identifier, name FROM projects;'
|
82
|
+
end
|
83
|
+
|
84
|
+
def pages_query
|
85
|
+
'SELECT id, title, wiki_id FROM wiki_pages;'
|
86
|
+
end
|
87
|
+
|
88
|
+
def wikis_query
|
89
|
+
'SELECT id, project_id FROM wikis;'
|
90
|
+
end
|
91
|
+
|
92
|
+
def pages
|
93
|
+
@pages ||= database.query(pages_query)
|
94
|
+
end
|
95
|
+
|
96
|
+
def projects
|
97
|
+
@projects ||= database.query(projects_query)
|
98
|
+
end
|
99
|
+
|
100
|
+
def wikis
|
101
|
+
@wikis ||= database.query(wikis_query)
|
102
|
+
end
|
103
|
+
|
104
|
+
def build_properties(title, row_content)
|
105
|
+
author = authors.fetch(row_content["author_id"], nil)
|
106
|
+
{
|
107
|
+
id: row_content['id'],
|
108
|
+
title: title,
|
109
|
+
body: row_content['data'],
|
110
|
+
markup: :textile,
|
111
|
+
latest: false,
|
112
|
+
time: row_content['updated_on'],
|
113
|
+
message: row_content['comments'],
|
114
|
+
author: author,
|
115
|
+
author_name: author.name
|
116
|
+
}
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module Caramelize
|
2
|
+
module InputWiki
|
3
|
+
class Wiki
|
4
|
+
include DatabaseConnector
|
5
|
+
|
6
|
+
attr_accessor :revisions, :wiki_title, :titles, :description, :namespaces, :options
|
7
|
+
|
8
|
+
def initialize(options={})
|
9
|
+
@options = options
|
10
|
+
@options[:filters] = []
|
11
|
+
@namespaces = []
|
12
|
+
end
|
13
|
+
|
14
|
+
def revisions_by_title(title)
|
15
|
+
# new array only containing pages by this name sorted by time asc
|
16
|
+
# this does not support renamed pages
|
17
|
+
revisions.select { |revision| revision.title == title }
|
18
|
+
.sort { |x,y| x.time <=> y.time }
|
19
|
+
end
|
20
|
+
|
21
|
+
# return an empty array in case this action was not overridden
|
22
|
+
def read_authors
|
23
|
+
[]
|
24
|
+
end
|
25
|
+
|
26
|
+
def namespaces
|
27
|
+
@namespaces ||= {}
|
28
|
+
end
|
29
|
+
|
30
|
+
def authors
|
31
|
+
@authors ||= {}
|
32
|
+
end
|
33
|
+
|
34
|
+
def revisions
|
35
|
+
@revisions ||= []
|
36
|
+
end
|
37
|
+
|
38
|
+
def titles
|
39
|
+
@titles ||= []
|
40
|
+
end
|
41
|
+
|
42
|
+
def convert_markup?(to_markup)
|
43
|
+
markup != to_markup
|
44
|
+
end
|
45
|
+
|
46
|
+
def filters
|
47
|
+
@options[:filters]
|
48
|
+
end
|
49
|
+
|
50
|
+
def latest_revisions
|
51
|
+
@latest_revisions ||= titles.map { |title| revisions_by_title(title).last }.compact
|
52
|
+
end
|
53
|
+
|
54
|
+
def markup
|
55
|
+
@options[:markup]
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'caramelize/database_connector'
|
2
|
+
require 'caramelize/filters/wikka_to_markdown'
|
3
|
+
|
4
|
+
module Caramelize
|
5
|
+
module InputWiki
|
6
|
+
class WikkaWiki < Wiki
|
7
|
+
include DatabaseConnector
|
8
|
+
|
9
|
+
SQL_PAGES = 'SELECT id, tag, body, time, latest, user, note FROM wikka_pages ORDER BY time;'.freeze
|
10
|
+
SQL_AUTHORS = 'SELECT name, email FROM wikka_users;'.freeze
|
11
|
+
|
12
|
+
def initialize(options = {})
|
13
|
+
super(options)
|
14
|
+
@options[:markup] = :wikka
|
15
|
+
@options[:filters] << Caramelize::Wikka2Markdown
|
16
|
+
end
|
17
|
+
|
18
|
+
# after calling this action, titles and @revisions are expected to be filled
|
19
|
+
def read_pages
|
20
|
+
pages.each do |row|
|
21
|
+
titles << row['tag']
|
22
|
+
page = Page.new(build_properties(row))
|
23
|
+
revisions << page
|
24
|
+
end
|
25
|
+
titles.uniq!
|
26
|
+
#revisions.sort! { |a,b| a.time <=> b.time }
|
27
|
+
|
28
|
+
revisions
|
29
|
+
end
|
30
|
+
|
31
|
+
def read_authors
|
32
|
+
results = database.query(authors_query)
|
33
|
+
results.each do |row|
|
34
|
+
authors[row['name']] = OpenStruct.new(name: row['name'],
|
35
|
+
email: row['email'] )
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def pages_query
|
42
|
+
SQL_PAGES
|
43
|
+
end
|
44
|
+
|
45
|
+
def authors_query
|
46
|
+
SQL_AUTHORS
|
47
|
+
end
|
48
|
+
|
49
|
+
def pages
|
50
|
+
@pages ||= database.query(pages_query)
|
51
|
+
end
|
52
|
+
|
53
|
+
def build_properties(row)
|
54
|
+
author = authors[row['user']]
|
55
|
+
{
|
56
|
+
id: row["id"],
|
57
|
+
title: row["tag"],
|
58
|
+
body: row["body"],
|
59
|
+
markup: :wikka,
|
60
|
+
latest: row["latest"] == "Y",
|
61
|
+
time: row["time"],
|
62
|
+
message: row["note"],
|
63
|
+
author: author,
|
64
|
+
author_name: row["user"]
|
65
|
+
}
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|