caramelize 1.2.0 → 1.3.0

data/lib/caramelize/filters/add_newline_to_page_end.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Caramelize
+  class AddNewlineToPageEnd
+    def initialize(body)
+      @body = body
+    end
+
+    # take an input stream and convert all wikka syntax to markdown syntax
+    def run
+      return @body if @body[@body.length - 1] == "\n"
+
+      migrated_body = @body.dup
+
+      migrated_body << "\n"
+
+      migrated_body
+    end
+  end
+end

data/lib/caramelize/filters/camel_case_to_wiki_links.rb

@@ -18,8 +18,8 @@ module Caramelize
     private
 
     def format_link(link)
-      link.gsub!(' ', '_')
-      link.gsub!('.', '')
+      link.tr!(' ', '_')
+      link.delete!('.')
       "[[#{link}]]"
     end
   end

data/lib/caramelize/filters/mediawiki_to_markdown.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'paru/pandoc'
+
+module Caramelize
+  class MediawikiToMarkdown
+    attr_reader :source_body
+
+    def initialize(source_body)
+      @source_body = source_body
+    end
+
+    def run
+      Paru::Pandoc.new do
+        from 'mediawiki'
+        to 'markdown'
+        markdown_headings 'atx'
+      end << source_body.dup
+    end
+  end
+end

data/lib/caramelize/filters/swap_wiki_links.rb

@@ -20,8 +20,8 @@ module Caramelize
 
     def format_link(label, link)
       link.downcase!
-      link.gsub!(' ', '_')
-      link.gsub!('.', '')
+      link.tr!(' ', '_')
+      link.delete!('.')
       "[[#{label}|#{link}]]"
     end
   end

data/lib/caramelize/filters/wikka_to_markdown.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 module Caramelize
-  class Wikka2Markdown
+  # WikkaWiki formatting rules: http://docs.wikkawiki.org/FormattingRules
+  class WikkaToMarkdown
     attr_reader :source_body
 
     def initialize(source_body)
@@ -9,9 +10,6 @@ module Caramelize
     end
 
     def run
-      # TODO: images: ({{image)(url\=?)?(.*)(}})
-      # markdown: ![Tux, the Linux mascot](/assets/images/tux.png)
-
       replace_headlines
       replace_formatting
       replace_lists
@@ -19,6 +17,7 @@ module Caramelize
       replace_links
       replace_inline_code
       replace_code_block
+      replace_images
 
       target_body
     end
@@ -43,8 +42,7 @@ module Caramelize
       target_body.gsub!(/(~-\s?)(.*)/, '- \2')     # unordered list
       target_body.gsub!(/(    -\s?)(.*)/, '- \2')     # unordered list
 
-      target_body.gsub!(/(~1\)\s?)(.*)/, '1. \2')     # unordered list
-      # TODO ordered lists
+      target_body.gsub!(/(~1\)\s?)(.*)/, '1. \2')     # ordered list
     end
 
     def replace_wiki_links
@@ -66,6 +64,17 @@ module Caramelize
       target_body.gsub!(/^%%\s(.*?)%%\s?/m) { "```\n#{::Regexp.last_match(1)}```\n" }
     end
 
+    def replace_images
+      # {{image class="center" alt="DVD logo" title="An image link" url="images/dvdvideo.gif" link="RecentChanges"}}
+      target_body.gsub!(/{{image\s(.*)}}/) do |image_match|
+        url = image_match.match(/url="([^"]*)"/)[1]
+        link = image_match.match(/link="([^"]*)"/) && image_match.match(/link="([^"]*)"/)[1]
+        alt = image_match.match(/alt="([^"]*)"/) && image_match.match(/alt="([^"]*)"/)[1]
+
+        link.nil? ? "![#{alt}](#{url})" : "[[<img src=\"#{url}\" alt=\"#{alt}\">|#{link}]]"
+      end
+    end
+
     def target_body
       @target_body ||= source_body.dup
     end

data/lib/caramelize/health_checks/home_page_check.rb

@@ -15,7 +15,7 @@ module Caramelize
 
       private
 
-      def has_home_page?
+      def has_home_page? # rubocop:todo Naming/PredicateName
         gollum.file('Home')
       end
     end

data/lib/caramelize/input_wiki/media_wiki.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'caramelize/filters/add_newline_to_page_end'
+require 'caramelize/filters/mediawiki_to_markdown'
+
+module Caramelize
+  module InputWiki
+    class MediaWiki < Wiki
+      include DatabaseConnector
+
+      # for Mediawiki v1.41
+      SQL_PAGES = %{
+        SELECT *
+        FROM revision as r
+        JOIN page as p on r.rev_page = p.page_id
+        JOIN comment as c ON c.comment_id = r.rev_comment_id
+        JOIN actor as a ON a.actor_id = r.rev_actor
+        JOIN slots as s ON s.slot_revision_id = r.rev_id
+        JOIN content as cn ON cn.`content_id` = s.slot_content_id
+        JOIN text as t ON cn.content_address = CONCAT('tt:', t.old_id)
+        ORDER BY r.rev_timestamp ASC;
+      }
+      SQL_AUTHORS = 'SELECT user_id, user_name, user_real_name, user_email FROM user;'
+      NAMESPACE_MAPPING = {
+        0 => :NS_MAIN,
+        1 => :NS_TALK,
+        2 => :NS_USER,
+        3 => :NS_USER_TALK,
+        4 => :NS_PROJECT,
+        5 => :NS_PROJECT_TALK,
+        6 => :NS_FILE,
+        7 => :NS_FILE_TALK,
+        8 => :NS_MEDIAWIKI,
+        9 => :NS_MEDIAWIKI_TALK,
+        10 => :NS_TEMPLATE,
+        11 => :NS_TEMPLATE_TALK,
+        12 => :NS_HELP,
+        13 => :NS_HELP_TALK,
+        14 => :NS_CATEGORY,
+        15 => :NS_CATEGORY_TALK
+      }.freeze
+
+      def initialize(options = {})
+        super(options)
+        @options[:markup] = :media_wiki
+        @options[:filters] << Caramelize::AddNewlineToPageEnd
+        @options[:filters] << ::Caramelize::MediawikiToMarkdown
+      end
+
+      # after calling this action, titles and @revisions are expected to be filled
+      def read_pages
+        pages.each do |row|
+          titles << row['page_title']
+          revisions << Page.new(build_properties(row))
+        end
+        titles.uniq!
+        revisions
+      end
+
+      def read_authors
+        database.query(authors_query).each do |row|
+          name = row['user_real_name'].empty? ? row['user_name'] : 'Anonymous'
+          email = row['user_email'].empty? ? nil : row['user_email']
+          authors[row['user_id']] = { name:, email: }
+        end
+
+        authors
+      end
+
+      def excluded_pages
+        []
+      end
+
+      private
+
+      def pages_query
+        SQL_PAGES
+      end
+
+      def authors_query
+        SQL_AUTHORS
+      end
+
+      def pages
+        @pages ||= database.query(pages_query)
+      end
+
+      def build_properties(row)
+        author = authors[row['actor_user']] || { name: 'Anonymous', email: 'anonymous@example.com' }
+        {
+          id: row['rev_id'],
+          title: title_by_namespace(row),
+          body: row['old_text'],
+          markup: :media_wiki,
+          latest: row['page_latest'] == row['rev_id'],
+          time: Time.strptime(row['rev_timestamp'], '%Y%m%d%H%M%S'),
+          message: row['comment_text'],
+          author:
+        }
+      end
+
+      def title_by_namespace(row)
+        return row['page_title'] if namespace_matches?(row['page_namespace'], :NS_MAIN)
+        return "#{row['page_title']}_Discussion" if namespace_matches?(row['page_namespace'], :NS_TALK)
+
+        row['page_title']
+      end
+
+      def namespace_matches?(namespace_id, expected_namespace)
+        NAMESPACE_MAPPING[namespace_id] == expected_namespace
+      end
+    end
+  end
+end

data/lib/caramelize/input_wiki/redmine_wiki.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'caramelize/input_wiki/wiki'
 require 'caramelize/filters/swap_wiki_links'
 require 'caramelize/filters/remove_table_tab_line_endings'
 
@@ -12,8 +11,8 @@ module Caramelize
       def initialize(options = {})
         super(options)
         @options[:markup] = :textile
-        @options[:filters] << Caramelize::SwapWikiLinks
-        @options[:filters] << Caramelize::RemoveTableTabLineEndings
+        @options[:filters] << ::Caramelize::SwapWikiLinks
+        @options[:filters] << ::Caramelize::RemoveTableTabLineEndings
         @options[:create_namespace_overview] = true
       end
 
@@ -25,14 +24,13 @@ module Caramelize
           build_page(row_page)
         end
         titles.uniq!
-        revisions.sort! { |a, b| a.time <=> b.time }
+        revisions.sort_by!(&:time)
 
         revisions
       end
 
       def read_authors
-        results = database.query(authors_query)
-        results.each do |row|
+        database.query(authors_query).each do |row|
           authors[row['id']] = { id: row['id'],
                                  name: row['login'],
                                  email: row['mail'] }
@@ -45,12 +43,12 @@ module Caramelize
       def build_page(row_page)
         results_contents = database.query(single_page_query(row_page['id']))
 
-        wiki = wikis.select { |row| row['id'] == row_page['wiki_id'] }.first
+        wiki = wikis.find { |row| row['id'] == row_page['wiki_id'] }
 
         project_identifier = ''
 
         if wiki
-          project = projects.select { |row| row['id'] == wiki['project_id'] }.first
+          project = projects.find { |row| row['id'] == wiki['project_id'] }
           project_identifier = "#{project['identifier']}/"
         end
 

data/lib/caramelize/input_wiki/wiki.rb

@@ -17,7 +17,7 @@ module Caramelize
         # new array only containing pages by this name sorted by time asc
         # this does not support renamed pages
         revisions.select { |revision| revision.title == title }
-                 .sort { |x, y| x.time <=> y.time }
+                 .sort_by(&:time)
       end
 
       # return an empty array in case this action was not overridden
@@ -25,7 +25,7 @@ module Caramelize
         []
       end
 
-      def namespaces
+      def namespaces # rubocop:todo Lint/DuplicateMethods
         @namespaces ||= {}
       end
 
@@ -33,11 +33,11 @@ module Caramelize
         @authors ||= {}
       end
 
-      def revisions
+      def revisions # rubocop:todo Lint/DuplicateMethods
         @revisions ||= []
       end
 
-      def titles
+      def titles # rubocop:todo Lint/DuplicateMethods
         @titles ||= []
       end
 
@@ -54,7 +54,7 @@ module Caramelize
       end
 
       def latest_revisions
-        @latest_revisions ||= titles.map { |title| revisions_by_title(title).last }.compact
+        @latest_revisions ||= titles.filter_map { |title| revisions_by_title(title).last }
       end
 
       def markup

data/lib/caramelize/input_wiki/{wikkawiki.rb → wikka_wiki.rb}

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
-require 'caramelize/database_connector'
-require 'caramelize/filters/add_newline_on_page_end'
+require 'caramelize/filters/add_newline_to_page_end'
 require 'caramelize/filters/camel_case_to_wiki_links'
 require 'caramelize/filters/wikka_to_markdown'
 
@@ -12,26 +11,23 @@ module Caramelize
 
       SQL_PAGES = 'SELECT id, tag, body, time, latest, user, note FROM wikka_pages ORDER BY time;'
       SQL_AUTHORS = 'SELECT name, email FROM wikka_users;'
-      FUNCTION_PAGES = %w[AdminBadWords AdminPages AdminUsers AdminSpamLog Callbacks CategoryAdmin CategoryCategory CategoryWiki DatabaseInfo FormattingRules HighScores InterWiki MyChanges MyPages OrphanedPages OwnedPages PageIndex PasswordForgotten RecentChanges RecentlyCommented Sandbox SysInfo TextSearch TextSearchExpanded UserSettings WantedPages WikiCategory WikkaInstaller WikkaConfig WikkaDocumentation WikkaMenulets WikkaReleaseNotes].freeze
+      FUNCTION_PAGES = %w[AdminBadWords AdminPages AdminUsers AdminSpamLog CategoryAdmin CategoryCategory CategoryWiki DatabaseInfo FormattingRules HighScores InterWiki MyChanges MyPages OrphanedPages OwnedPages PageIndex PasswordForgotten RecentChanges RecentlyCommented Sandbox SysInfo TableMarkup TableMarkupReference TextSearch TextSearchExpanded UserSettings WantedPages WikiCategory WikkaInstaller WikkaConfig WikkaDocumentation WikkaMenulets WikkaReleaseNotes].freeze
 
       def initialize(options = {})
         super(options)
         @options[:markup] = :wikka
-        @options[:filters] << Caramelize::AddNewlineOnPageEnd
-        @options[:filters] << Caramelize::Wikka2Markdown
-        @options[:filters] << Caramelize::CamelCaseToWikiLinks
+        @options[:filters] << ::Caramelize::AddNewlineToPageEnd
+        @options[:filters] << ::Caramelize::WikkaToMarkdown
+        @options[:filters] << ::Caramelize::CamelCaseToWikiLinks
       end
 
       # after calling this action, titles and @revisions are expected to be filled
       def read_pages
         pages.each do |row|
           titles << row['tag']
-          page = Page.new(build_properties(row))
-          revisions << page
+          revisions << Page.new(build_properties(row))
         end
         titles.uniq!
-        # revisions.sort! { |a,b| a.time <=> b.time }
-
         revisions
       end
 

data/lib/caramelize/output_wiki/gollum.rb

@@ -38,7 +38,7 @@ module Caramelize
       def commit_history(revisions, options = {}, &block)
         revisions.each_with_index do |page, index|
           # call debug output from outside
-          block.call(page, index) if block_given?
+          yield(page, index) if block
           commit_revision(page, options.fetch(:markup, :markdown))
         end
       end

data/lib/caramelize/page.rb

@@ -23,7 +23,7 @@ module Caramelize
       author[:name]
     end
 
-    def author
+    def author # rubocop:todo Lint/DuplicateMethods
       @author ||= { name: 'Caramelize', email: 'mail@example.com' }
     end
 

data/lib/caramelize/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Caramelize
-  VERSION = '1.2.0'
+  VERSION = '1.3.0'
 end

data/lib/caramelize.rb

@@ -2,18 +2,20 @@
 
 require 'caramelize/version'
 require 'caramelize/page'
-require 'caramelize/content_transferer'
-require 'caramelize/filter_processor'
 require 'caramelize/database_connector'
+require 'caramelize/filters/add_newline_to_page_end'
+require 'caramelize/filter_processor'
+require 'caramelize/input_wiki/wiki'
+require 'caramelize/input_wiki/media_wiki'
+require 'caramelize/input_wiki/redmine_wiki'
+require 'caramelize/input_wiki/wikka_wiki'
+require 'caramelize/content_transferer'
 require 'caramelize/health_check'
 require 'caramelize/health_checks/home_page_check'
 require 'caramelize/health_checks/orphaned_pages_check'
 require 'caramelize/health_checks/page'
 require 'caramelize/output_wiki/gollum'
 require 'caramelize/services/page_builder'
-require 'caramelize/input_wiki/wiki'
-require 'caramelize/input_wiki/redmine_wiki'
-require 'caramelize/input_wiki/wikkawiki'
 
 Encoding.default_external = Encoding::UTF_8
 Encoding.default_internal = Encoding::UTF_8

data/samples/media_wiki.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Connecting to the MediaWiki Docker setup  at https://hub.docker.com/_/mediawiki/
+
+def input_wiki
+  options = {
+    host: '0.0.0.0',
+    username: 'wikiuser',
+    password: 'example',
+    database: 'my_wiki',
+    port: 63_645
+  }
+  Caramelize::InputWiki::MediaWiki.new(options)
+end

data/spec/lib/caramelize/filter_processor_spec.rb

@@ -6,18 +6,20 @@ describe Caramelize::FilterProcessor do
   subject(:processor) { described_class.new(input_wiki) }
 
   let(:filters) { [] }
-  let(:input_wiki) { double(filters:) }
+  let(:input_wiki) { double(filters:) } # rubocop:todo RSpec/VerifiedDoubles
   let(:body) { 'body' }
 
-  class ReverseFilter
+  # rubocop:todo RSpec/LeakyConstantDeclaration
+  class ReverseFilter # rubocop:todo Lint/ConstantDefinitionInBlock, RSpec/LeakyConstantDeclaration
     def initialize(body)
       @body = body
     end
 
     def run
-      @body.reverse
+      @body.reverse # rubocop:todo RSpec/InstanceVariable
     end
   end
+  # rubocop:enable RSpec/LeakyConstantDeclaration
 
   describe '#run' do
     context 'without any filters' do

data/spec/lib/caramelize/filters/{add_newline_on_page_end_spec.rb → add_newline_to_page_end_spec.rb}

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-describe Caramelize::AddNewlineOnPageEnd do
+describe Caramelize::AddNewlineToPageEnd do
   describe '#run' do
     subject(:run) { filter.run }
 

data/spec/lib/caramelize/filters/camel_case_to_wiki_links_spec.rb

@@ -2,7 +2,9 @@
 
 require 'spec_helper'
 
-describe Caramelize::CamelCaseToWikiLinks do
+# rubocop:todo RSpec/SpecFilePathFormat
+describe Caramelize::CamelCaseToWikiLinks do # rubocop:todo RSpec/FilePath, RSpec/SpecFilePathFormat
+  # rubocop:enable RSpec/SpecFilePathFormat
   describe '#run' do
     subject(:run) { filter.run }
 

data/spec/lib/caramelize/filters/mediawiki_to_markdown_spec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Caramelize::MediawikiToMarkdown do
+  let(:filter) { described_class.new(body) }
+
+  describe '#run' do
+    subject { filter.run }
+
+    # the conversions are performed through pandoc-ruby.
+    # This is a smoke test to see if it's called at all,
+    # but not testing every single option.
+
+    context 'when headline h1' do
+      let(:body) { "= Headline =\n" }
+
+      it { is_expected.to eq "# Headline\n" }
+    end
+
+    context 'when headline h2' do
+      let(:body) { "== Headline ==\n" }
+
+      it { is_expected.to eq "## Headline\n" }
+    end
+  end
+end

data/spec/lib/caramelize/filters/remove_table_tab_line_endings_spec.rb

@@ -2,13 +2,15 @@
 
 require 'spec_helper'
 
-describe Caramelize::RemoveTableTabLineEndings do
+# rubocop:todo RSpec/SpecFilePathFormat
+describe Caramelize::RemoveTableTabLineEndings do # rubocop:todo RSpec/FilePath, RSpec/SpecFilePathFormat
+  # rubocop:enable RSpec/SpecFilePathFormat
   subject(:run) { filter.run }
 
   let(:filter) { described_class.new(body) }
 
   describe '#run' do
-    context 'table with tabs at unix line-endings' do
+    context 'table with tabs at unix line-endings' do # rubocop:todo RSpec/ContextWording
       let(:body) { "cell1\t|cell2\t|\t\t\n" }
 
       it 'removes tabs at end of line' do
@@ -23,7 +25,7 @@ describe Caramelize::RemoveTableTabLineEndings do
         expect(run).to eq "cell1\t|cell2\t|\n"
       end
 
-      context 'replace in full file' do
+      context 'replace in full file' do # rubocop:todo RSpec/ContextWording
         let(:body) do
           File.read(File.join(['spec', 'fixtures', 'markup', 'table-tab-line-endings-input.textile']))
         end

data/spec/lib/caramelize/filters/swap_wiki_links_spec.rb

@@ -2,7 +2,9 @@
 
 require 'spec_helper'
 
-describe Caramelize::SwapWikiLinks do
+# rubocop:todo RSpec/SpecFilePathFormat
+describe Caramelize::SwapWikiLinks do # rubocop:todo RSpec/FilePath, RSpec/SpecFilePathFormat
+  # rubocop:enable RSpec/SpecFilePathFormat
   describe '#run' do
     subject(:run) { filter.run }