caramelize 0.1.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +21 -53
  3. data/.travis.yml +5 -0
  4. data/CODE_OF_CONDUCT.md +76 -0
  5. data/Gemfile +3 -3
  6. data/LICENSE.md +1 -1
  7. data/README.md +48 -46
  8. data/Rakefile +8 -7
  9. data/bin/caramelize +73 -10
  10. data/caramelize.gemspec +28 -24
  11. data/lib/caramelize.rb +14 -0
  12. data/lib/caramelize/caramel.rb +48 -44
  13. data/lib/caramelize/content_transferer.rb +126 -69
  14. data/lib/caramelize/database_connector.rb +3 -6
  15. data/lib/caramelize/filter_processor.rb +27 -0
  16. data/lib/caramelize/filters/remove_table_tab_line_endings.rb +15 -0
  17. data/lib/caramelize/filters/swap_wiki_links.rb +26 -0
  18. data/lib/caramelize/filters/wikka_to_markdown.rb +67 -0
  19. data/lib/caramelize/health_check.rb +85 -0
  20. data/lib/caramelize/input_wiki/redmine_wiki.rb +120 -0
  21. data/lib/caramelize/input_wiki/wiki.rb +59 -0
  22. data/lib/caramelize/input_wiki/wikkawiki.rb +69 -0
  23. data/lib/caramelize/output_wiki/gollum.rb +80 -0
  24. data/lib/caramelize/page.rb +38 -14
  25. data/lib/caramelize/services/page_builder.rb +20 -0
  26. data/lib/caramelize/version.rb +1 -1
  27. data/spec/fixtures/markup/swap-links-input.textile +57 -0
  28. data/spec/fixtures/markup/swap-links-output.textile +57 -0
  29. data/spec/fixtures/markup/table-tab-line-endings-input.textile +145 -0
  30. data/spec/fixtures/markup/table-tab-line-endings-output.textile +145 -0
  31. data/spec/lib/caramelize/content_transferer_spec.rb +9 -0
  32. data/spec/lib/caramelize/filter_processor_spec.rb +34 -0
  33. data/spec/lib/caramelize/filters/remove_table_tab_line_endings_spec.rb +49 -0
  34. data/spec/lib/caramelize/filters/swap_wiki_links_spec.rb +49 -0
  35. data/spec/lib/caramelize/filters/wikka_to_markdown_spec.rb +198 -0
  36. data/spec/lib/caramelize/input_wiki/wiki_spec.rb +57 -0
  37. data/spec/lib/caramelize/output_wiki/gollum_spec.rb +113 -0
  38. data/spec/lib/caramelize/page_spec.rb +67 -0
  39. data/spec/lib/caramelize/services/page_builder.rb +29 -0
  40. data/spec/spec_helper.rb +8 -0
  41. metadata +165 -54
  42. data/lib/caramelize/author.rb +0 -8
  43. data/lib/caramelize/cli.rb +0 -80
  44. data/lib/caramelize/cli/create_command.rb +0 -52
  45. data/lib/caramelize/cli/run_command.rb +0 -33
  46. data/lib/caramelize/ext.rb +0 -17
  47. data/lib/caramelize/gollum_output.rb +0 -56
  48. data/lib/caramelize/wiki/redmine_wiki.rb +0 -60
  49. data/lib/caramelize/wiki/trac_converter.rb +0 -82
  50. data/lib/caramelize/wiki/wiki.rb +0 -41
  51. data/lib/caramelize/wiki/wikka_converter.rb +0 -38
  52. data/lib/caramelize/wiki/wikkawiki.rb +0 -55
  53. data/test/helper.rb +0 -18
  54. data/test/test_caramelize.rb +0 -7
@@ -0,0 +1,15 @@
1
+ module Caramelize
2
+ class RemoveTableTabLineEndings
3
+
4
+ def initialize(body)
5
+ @body = body
6
+ end
7
+
8
+ # take an input stream and convert all wikka syntax to markdown syntax
9
+ def run
10
+ migrated_body = @body.dup
11
+ migrated_body.gsub!(/\|[\t ]*\r?[\n]/, "|\n")
12
+ migrated_body
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,26 @@
1
+ module Caramelize
2
+ class SwapWikiLinks
3
+ def initialize(body)
4
+ @body = body
5
+ end
6
+
7
+ # take an input stream and convert all wikka syntax to markdown syntax
8
+ def run
9
+ migrated_body = @body.dup
10
+
11
+ migrated_body.gsub!(/\[\[(\S+)\|(.+?)\]\]/) { format_link($2, $1) }
12
+ migrated_body.gsub!(/\[\[([\w\s\-\.]*)\]\]/) { format_link($1, $1.dup) }
13
+
14
+ migrated_body
15
+ end
16
+
17
+ private
18
+
19
+ def format_link(label, link)
20
+ link.downcase!
21
+ link.gsub!(' ', '_')
22
+ link.gsub!(/\./, '')
23
+ "[[#{label}|#{link}]]"
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,67 @@
1
+ module Caramelize
2
+ class Wikka2Markdown
3
+ attr_reader :source_body
4
+
5
+ def initialize(source_body)
6
+ @source_body = source_body
7
+ end
8
+
9
+ def run
10
+ # TODO images: ({{image)(url\=?)?(.*)(}})
11
+ # markdown: ![Tux, the Linux mascot](/assets/images/tux.png)
12
+
13
+ replace_headlines
14
+ replace_formatting
15
+ replace_lists
16
+ replace_wiki_links
17
+ replace_links
18
+ replace_code_block
19
+
20
+ target_body
21
+ end
22
+
23
+ def replace_headlines
24
+ target_body.gsub!(/(======)(.*?)(======)/ ) {|s| '# ' + $2 } #h1
25
+ target_body.gsub!(/(=====)(.*?)(=====)/) {|s| '## ' + $2 } #h2
26
+ target_body.gsub!(/(====)(.*?)(====)/) {|s| '### ' + $2 } #h3
27
+ target_body.gsub!(/(===)(.*?)(===)/) {|s| '#### ' + $2 } #h4
28
+ target_body.gsub!(/(==)(.*?)(==)/) {|s| '##### ' + $2 } #h5
29
+ end
30
+
31
+ def replace_formatting
32
+ target_body.gsub!(/(\*\*)(.*?)(\*\*)/) {|s| '**' + $2 + '**' } #bold
33
+ target_body.gsub!(/(\/\/)(.*?)(\/\/)/, '*\2*') #italic
34
+ target_body.gsub!(/(__)(.*?)(__)/) {|s| '<u>' + $2 + '</u>'} #underline
35
+ target_body.gsub!(/(---)/, ' ') #forced linebreak
36
+ end
37
+
38
+ def replace_lists
39
+ target_body.gsub!(/(\t-\s?)(.*)/, '* \2') # unordered list
40
+ target_body.gsub!(/(~-\s?)(.*)/, '* \2') # unordered list
41
+ target_body.gsub!(/( -\s?)(.*)/, '* \2') # unordered list
42
+
43
+ target_body.gsub!(/(~1\)\s?)(.*)/, '1. \2') # unordered list
44
+ # TODO ordered lists
45
+ end
46
+
47
+ def replace_wiki_links
48
+ target_body.gsub!(/[\[]{2}(\w+)[\s|](.+?)[\]]{2}/, '[[\2|\1]]')
49
+ end
50
+
51
+ def replace_links
52
+ target_body.gsub!(/[\[]{2}((\w+):[\S][^\| ]*)[\| ](.*)[\]]{2}/,
53
+ '[\3](\1)')
54
+ target_body.gsub!(/[\[]{2}((\w+):.*)[\]]{2}/, '<\1>')
55
+ end
56
+
57
+ def replace_code_block
58
+ target_body.gsub!(/^%%\s(.*?)%%\s?/m) do
59
+ $1.gsub(/^/, ' ')
60
+ end
61
+ end
62
+
63
+ def target_body
64
+ @target_body ||= source_body.dup
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,85 @@
1
+ module Caramelize
2
+ class HealthCheck
3
+ attr_reader :wiki_path, :options
4
+
5
+ DEFAULT_GOLLUM_HOME_TITLE = 'Home'.freeze
6
+
7
+ def initialize(wiki_path, options={})
8
+ @wiki_path = wiki_path
9
+ @options = options
10
+ end
11
+
12
+ def execute
13
+ #puts page_paths.sort.inspect
14
+
15
+ check_pages
16
+
17
+ #puts intra_wiki_paths.sort.inspect
18
+
19
+ puts "\n # Pages not linked within Wiki:"
20
+ puts page_paths_without_intra_wiki_path.sort.inspect
21
+ end
22
+
23
+ private
24
+
25
+ def files
26
+ @files ||= Dir.glob([wiki_path, '**/*.md'].join('/'))
27
+ end
28
+
29
+ def file_names
30
+ files.map do |file|
31
+ file.gsub("#{wiki_path}/", '').split('.').first
32
+ end
33
+ end
34
+
35
+ def check_pages
36
+ pages.each do |page|
37
+ puts "\n## #{page.path}"
38
+ check_page(page)
39
+ end
40
+ end
41
+
42
+ def check_page(page)
43
+ intra_wiki_links = intra_wiki_links(page.text_data)
44
+ available = 0
45
+ intra_wiki_links.each do |link|
46
+ intra_wiki_link = page.path.split('/').first == page.path ? link : [page.path.split('/').first, link].join('/')
47
+ if !page_paths.include?(intra_wiki_link)
48
+ puts "#{intra_wiki_link} expected, but missing"
49
+ else
50
+ available += 1
51
+ intra_wiki_paths << intra_wiki_link
52
+ end
53
+ end
54
+ puts "#{available}/#{intra_wiki_links.count} available"
55
+ end
56
+
57
+ def intra_wiki_links(body)
58
+ body.scan(/\[\[(.+\|)?(\S+)\]\]/).map { |match| match[1] }.uniq
59
+ end
60
+
61
+ def pages
62
+ gollum.pages
63
+ end
64
+
65
+ def page_paths
66
+ pages.map(&:path).map { |path| path.split('.').first }
67
+ end
68
+
69
+ def intra_wiki_paths
70
+ @intra_wiki_paths ||= []
71
+ end
72
+
73
+ def page_paths_without_intra_wiki_path
74
+ page_paths - intra_wiki_paths
75
+ end
76
+
77
+ def check_home_page
78
+ puts "Home.md missing" if File.exist?('wiki-export/Home.md')
79
+ end
80
+
81
+ def gollum
82
+ @gollum ||= ::Gollum::Wiki.new(wiki_path)
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,120 @@
1
+ require 'caramelize/input_wiki/wiki'
2
+ require 'caramelize/filters/swap_wiki_links'
3
+ require 'caramelize/filters/remove_table_tab_line_endings'
4
+
5
+ module Caramelize
6
+ module InputWiki
7
+ class RedmineWiki < Wiki
8
+ include DatabaseConnector
9
+
10
+ def initialize(options = {})
11
+ super(options)
12
+ @options[:markup] = :textile
13
+ @options[:filters] << Caramelize::SwapWikiLinks
14
+ @options[:filters] << Caramelize::RemoveTableTabLineEndings
15
+ @options[:create_namespace_overview] = true
16
+ end
17
+
18
+ # after calling this action, I expect the titles and revisions to be filled
19
+ def read_pages
20
+ add_projects_as_namespaces
21
+
22
+ pages.each do |row_page|
23
+ build_page(row_page)
24
+ end
25
+ titles.uniq!
26
+ revisions.sort! { |a,b| a.time <=> b.time }
27
+
28
+ revisions
29
+ end
30
+
31
+ def read_authors
32
+ results = database.query(authors_query)
33
+ results.each do |row|
34
+ authors[row["id"]] = OpenStruct.new(id: row["id"],
35
+ name: row["login"],
36
+ email: row["mail"])
37
+ end
38
+ authors
39
+ end
40
+
41
+ private
42
+
43
+ def build_page(row_page)
44
+ results_contents = database.query(single_page_query(row_page['id']))
45
+
46
+ wiki = wikis.select{ |row| row['id'] == row_page['wiki_id'] }.first
47
+
48
+ project_identifier = ''
49
+
50
+ if wiki
51
+ project = projects.select{ |row| row['id'] == wiki['project_id'] }.first
52
+ project_identifier = project['identifier'] + '/'
53
+ end
54
+
55
+ title = project_identifier + row_page['title']
56
+ titles << title
57
+
58
+ results_contents.each do |row_content|
59
+ page = Page.new(build_properties(title, row_content))
60
+ revisions << page
61
+ end
62
+ end
63
+
64
+ def add_projects_as_namespaces
65
+ projects.each do |row_project|
66
+ namespace = OpenStruct.new(identifier: row_project['identifier'],
67
+ name: row_project['name'])
68
+ namespaces << namespace
69
+ end
70
+ end
71
+
72
+ def authors_query
73
+ 'SELECT id, login, mail FROM users;'
74
+ end
75
+
76
+ def single_page_query(page_id)
77
+ "SELECT * FROM wiki_content_versions WHERE page_id='#{page_id}' ORDER BY updated_on;"
78
+ end
79
+
80
+ def projects_query
81
+ 'SELECT id, identifier, name FROM projects;'
82
+ end
83
+
84
+ def pages_query
85
+ 'SELECT id, title, wiki_id FROM wiki_pages;'
86
+ end
87
+
88
+ def wikis_query
89
+ 'SELECT id, project_id FROM wikis;'
90
+ end
91
+
92
+ def pages
93
+ @pages ||= database.query(pages_query)
94
+ end
95
+
96
+ def projects
97
+ @projects ||= database.query(projects_query)
98
+ end
99
+
100
+ def wikis
101
+ @wikis ||= database.query(wikis_query)
102
+ end
103
+
104
+ def build_properties(title, row_content)
105
+ author = authors.fetch(row_content["author_id"], nil)
106
+ {
107
+ id: row_content['id'],
108
+ title: title,
109
+ body: row_content['data'],
110
+ markup: :textile,
111
+ latest: false,
112
+ time: row_content['updated_on'],
113
+ message: row_content['comments'],
114
+ author: author,
115
+ author_name: author.name
116
+ }
117
+ end
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,59 @@
1
+ module Caramelize
2
+ module InputWiki
3
+ class Wiki
4
+ include DatabaseConnector
5
+
6
+ attr_accessor :revisions, :wiki_title, :titles, :description, :namespaces, :options
7
+
8
+ def initialize(options={})
9
+ @options = options
10
+ @options[:filters] = []
11
+ @namespaces = []
12
+ end
13
+
14
+ def revisions_by_title(title)
15
+ # new array only containing pages by this name sorted by time asc
16
+ # this does not support renamed pages
17
+ revisions.select { |revision| revision.title == title }
18
+ .sort { |x,y| x.time <=> y.time }
19
+ end
20
+
21
+ # return an empty array in case this action was not overridden
22
+ def read_authors
23
+ []
24
+ end
25
+
26
+ def namespaces
27
+ @namespaces ||= {}
28
+ end
29
+
30
+ def authors
31
+ @authors ||= {}
32
+ end
33
+
34
+ def revisions
35
+ @revisions ||= []
36
+ end
37
+
38
+ def titles
39
+ @titles ||= []
40
+ end
41
+
42
+ def convert_markup?(to_markup)
43
+ markup != to_markup
44
+ end
45
+
46
+ def filters
47
+ @options[:filters]
48
+ end
49
+
50
+ def latest_revisions
51
+ @latest_revisions ||= titles.map { |title| revisions_by_title(title).last }.compact
52
+ end
53
+
54
+ def markup
55
+ @options[:markup]
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,69 @@
1
+ require 'caramelize/database_connector'
2
+ require 'caramelize/filters/wikka_to_markdown'
3
+
4
+ module Caramelize
5
+ module InputWiki
6
+ class WikkaWiki < Wiki
7
+ include DatabaseConnector
8
+
9
+ SQL_PAGES = 'SELECT id, tag, body, time, latest, user, note FROM wikka_pages ORDER BY time;'.freeze
10
+ SQL_AUTHORS = 'SELECT name, email FROM wikka_users;'.freeze
11
+
12
+ def initialize(options = {})
13
+ super(options)
14
+ @options[:markup] = :wikka
15
+ @options[:filters] << Caramelize::Wikka2Markdown
16
+ end
17
+
18
+ # after calling this action, titles and @revisions are expected to be filled
19
+ def read_pages
20
+ pages.each do |row|
21
+ titles << row['tag']
22
+ page = Page.new(build_properties(row))
23
+ revisions << page
24
+ end
25
+ titles.uniq!
26
+ #revisions.sort! { |a,b| a.time <=> b.time }
27
+
28
+ revisions
29
+ end
30
+
31
+ def read_authors
32
+ results = database.query(authors_query)
33
+ results.each do |row|
34
+ authors[row['name']] = OpenStruct.new(name: row['name'],
35
+ email: row['email'] )
36
+ end
37
+ end
38
+
39
+ private
40
+
41
+ def pages_query
42
+ SQL_PAGES
43
+ end
44
+
45
+ def authors_query
46
+ SQL_AUTHORS
47
+ end
48
+
49
+ def pages
50
+ @pages ||= database.query(pages_query)
51
+ end
52
+
53
+ def build_properties(row)
54
+ author = authors[row['user']]
55
+ {
56
+ id: row["id"],
57
+ title: row["tag"],
58
+ body: row["body"],
59
+ markup: :wikka,
60
+ latest: row["latest"] == "Y",
61
+ time: row["time"],
62
+ message: row["note"],
63
+ author: author,
64
+ author_name: row["user"]
65
+ }
66
+ end
67
+ end
68
+ end
69
+ end