caramelize 1.2.1 → 1.3.0

data/lib/caramelize/filters/mediawiki_to_markdown.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'paru/pandoc'
+
+module Caramelize
+  class MediawikiToMarkdown
+    attr_reader :source_body
+
+    def initialize(source_body)
+      @source_body = source_body
+    end
+
+    def run
+      Paru::Pandoc.new do
+        from 'mediawiki'
+        to 'markdown'
+        markdown_headings 'atx'
+      end << source_body.dup
+    end
+  end
+end

data/lib/caramelize/filters/swap_wiki_links.rb

@@ -20,8 +20,8 @@ module Caramelize
 
     def format_link(label, link)
       link.downcase!
-      link.gsub!(' ', '_')
-      link.gsub!('.', '')
+      link.tr!(' ', '_')
+      link.delete!('.')
       "[[#{label}|#{link}]]"
     end
   end

data/lib/caramelize/filters/wikka_to_markdown.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 module Caramelize
-  class Wikka2Markdown
+  # WikkaWiki formatting rules: http://docs.wikkawiki.org/FormattingRules
+  class WikkaToMarkdown
     attr_reader :source_body
 
     def initialize(source_body)
@@ -9,9 +10,6 @@ module Caramelize
     end
 
     def run
-      # TODO: images: ({{image)(url\=?)?(.*)(}})
-      # markdown: ![Tux, the Linux mascot](/assets/images/tux.png)
-
       replace_headlines
       replace_formatting
       replace_lists
@@ -19,6 +17,7 @@ module Caramelize
       replace_links
       replace_inline_code
       replace_code_block
+      replace_images
 
       target_body
     end
@@ -43,8 +42,7 @@ module Caramelize
       target_body.gsub!(/(~-\s?)(.*)/, '- \2')     # unordered list
       target_body.gsub!(/(    -\s?)(.*)/, '- \2')     # unordered list
 
-      target_body.gsub!(/(~1\)\s?)(.*)/, '1. \2')     # unordered list
-      # TODO ordered lists
+      target_body.gsub!(/(~1\)\s?)(.*)/, '1. \2')     # ordered list
     end
 
     def replace_wiki_links
@@ -66,6 +64,17 @@ module Caramelize
       target_body.gsub!(/^%%\s(.*?)%%\s?/m) { "```\n#{::Regexp.last_match(1)}```\n" }
     end
 
+    def replace_images
+      # {{image class="center" alt="DVD logo" title="An image link" url="images/dvdvideo.gif" link="RecentChanges"}}
+      target_body.gsub!(/{{image\s(.*)}}/) do |image_match|
+        url = image_match.match(/url="([^"]*)"/)[1]
+        link = image_match.match(/link="([^"]*)"/) && image_match.match(/link="([^"]*)"/)[1]
+        alt = image_match.match(/alt="([^"]*)"/) && image_match.match(/alt="([^"]*)"/)[1]
+
+        link.nil? ? "![#{alt}](#{url})" : "[[<img src=\"#{url}\" alt=\"#{alt}\">|#{link}]]"
+      end
+    end
+
     def target_body
       @target_body ||= source_body.dup
     end

data/lib/caramelize/health_checks/orphaned_pages_check.rb

@@ -18,7 +18,7 @@ module Caramelize
 
       private
 
-      def check_page(page) # rubocop:todo Metrics/MethodLength
+      def check_page(page)
         0.tap do |available_count|
           page.intra_wiki_links.each do |intra_wiki_path|
             if page_paths.include?(intra_wiki_path)

data/lib/caramelize/input_wiki/media_wiki.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'caramelize/filters/add_newline_to_page_end'
+require 'caramelize/filters/mediawiki_to_markdown'
+
+module Caramelize
+  module InputWiki
+    class MediaWiki < Wiki
+      include DatabaseConnector
+
+      # for Mediawiki v1.41
+      SQL_PAGES = %{
+        SELECT *
+        FROM revision as r
+        JOIN page as p on r.rev_page = p.page_id
+        JOIN comment as c ON c.comment_id = r.rev_comment_id
+        JOIN actor as a ON a.actor_id = r.rev_actor
+        JOIN slots as s ON s.slot_revision_id = r.rev_id
+        JOIN content as cn ON cn.`content_id` = s.slot_content_id
+        JOIN text as t ON cn.content_address = CONCAT('tt:', t.old_id)
+        ORDER BY r.rev_timestamp ASC;
+      }
+      SQL_AUTHORS = 'SELECT user_id, user_name, user_real_name, user_email FROM user;'
+      NAMESPACE_MAPPING = {
+        0 => :NS_MAIN,
+        1 => :NS_TALK,
+        2 => :NS_USER,
+        3 => :NS_USER_TALK,
+        4 => :NS_PROJECT,
+        5 => :NS_PROJECT_TALK,
+        6 => :NS_FILE,
+        7 => :NS_FILE_TALK,
+        8 => :NS_MEDIAWIKI,
+        9 => :NS_MEDIAWIKI_TALK,
+        10 => :NS_TEMPLATE,
+        11 => :NS_TEMPLATE_TALK,
+        12 => :NS_HELP,
+        13 => :NS_HELP_TALK,
+        14 => :NS_CATEGORY,
+        15 => :NS_CATEGORY_TALK
+      }.freeze
+
+      def initialize(options = {})
+        super(options)
+        @options[:markup] = :media_wiki
+        @options[:filters] << Caramelize::AddNewlineToPageEnd
+        @options[:filters] << ::Caramelize::MediawikiToMarkdown
+      end
+
+      # after calling this action, titles and @revisions are expected to be filled
+      def read_pages
+        pages.each do |row|
+          titles << row['page_title']
+          revisions << Page.new(build_properties(row))
+        end
+        titles.uniq!
+        revisions
+      end
+
+      def read_authors
+        database.query(authors_query).each do |row|
+          name = row['user_real_name'].empty? ? row['user_name'] : 'Anonymous'
+          email = row['user_email'].empty? ? nil : row['user_email']
+          authors[row['user_id']] = { name:, email: }
+        end
+
+        authors
+      end
+
+      def excluded_pages
+        []
+      end
+
+      private
+
+      def pages_query
+        SQL_PAGES
+      end
+
+      def authors_query
+        SQL_AUTHORS
+      end
+
+      def pages
+        @pages ||= database.query(pages_query)
+      end
+
+      def build_properties(row)
+        author = authors[row['actor_user']] || { name: 'Anonymous', email: 'anonymous@example.com' }
+        {
+          id: row['rev_id'],
+          title: title_by_namespace(row),
+          body: row['old_text'],
+          markup: :media_wiki,
+          latest: row['page_latest'] == row['rev_id'],
+          time: Time.strptime(row['rev_timestamp'], '%Y%m%d%H%M%S'),
+          message: row['comment_text'],
+          author:
+        }
+      end
+
+      def title_by_namespace(row)
+        return row['page_title'] if namespace_matches?(row['page_namespace'], :NS_MAIN)
+        return "#{row['page_title']}_Discussion" if namespace_matches?(row['page_namespace'], :NS_TALK)
+
+        row['page_title']
+      end
+
+      def namespace_matches?(namespace_id, expected_namespace)
+        NAMESPACE_MAPPING[namespace_id] == expected_namespace
+      end
+    end
+  end
+end

data/lib/caramelize/input_wiki/redmine_wiki.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'caramelize/input_wiki/wiki'
 require 'caramelize/filters/swap_wiki_links'
 require 'caramelize/filters/remove_table_tab_line_endings'
 
@@ -12,8 +11,8 @@ module Caramelize
       def initialize(options = {})
         super(options)
         @options[:markup] = :textile
-        @options[:filters] << Caramelize::SwapWikiLinks
-        @options[:filters] << Caramelize::RemoveTableTabLineEndings
+        @options[:filters] << ::Caramelize::SwapWikiLinks
+        @options[:filters] << ::Caramelize::RemoveTableTabLineEndings
         @options[:create_namespace_overview] = true
       end
 
@@ -25,14 +24,13 @@ module Caramelize
           build_page(row_page)
         end
         titles.uniq!
-        revisions.sort! { |a, b| a.time <=> b.time }
+        revisions.sort_by!(&:time)
 
         revisions
       end
 
       def read_authors
-        results = database.query(authors_query)
-        results.each do |row|
+        database.query(authors_query).each do |row|
           authors[row['id']] = { id: row['id'],
                                  name: row['login'],
                                  email: row['mail'] }
@@ -42,16 +40,15 @@ module Caramelize
 
       private
 
-      # rubocop:todo Metrics/MethodLength
-      def build_page(row_page) # rubocop:todo Metrics/AbcSize, Metrics/MethodLength
+      def build_page(row_page)
         results_contents = database.query(single_page_query(row_page['id']))
 
-        wiki = wikis.select { |row| row['id'] == row_page['wiki_id'] }.first
+        wiki = wikis.find { |row| row['id'] == row_page['wiki_id'] }
 
         project_identifier = ''
 
         if wiki
-          project = projects.select { |row| row['id'] == wiki['project_id'] }.first
+          project = projects.find { |row| row['id'] == wiki['project_id'] }
           project_identifier = "#{project['identifier']}/"
         end
 
@@ -63,7 +60,6 @@ module Caramelize
           revisions << page
         end
       end
-      # rubocop:enable Metrics/MethodLength
 
       def add_projects_as_namespaces
         projects.each do |row_project|
@@ -105,7 +101,7 @@ module Caramelize
         @wikis ||= database.query(wikis_query)
       end
 
-      def build_properties(title, row_content) # rubocop:todo Metrics/MethodLength
+      def build_properties(title, row_content)
         author = authors.fetch(row_content['author_id'], nil)
         {
           id: row_content['id'],

data/lib/caramelize/input_wiki/wiki.rb

@@ -17,7 +17,7 @@ module Caramelize
         # new array only containing pages by this name sorted by time asc
         # this does not support renamed pages
         revisions.select { |revision| revision.title == title }
-                 .sort { |x, y| x.time <=> y.time }
+                 .sort_by(&:time)
       end
 
       # return an empty array in case this action was not overridden
@@ -54,7 +54,7 @@ module Caramelize
       end
 
       def latest_revisions
-        @latest_revisions ||= titles.map { |title| revisions_by_title(title).last }.compact
+        @latest_revisions ||= titles.filter_map { |title| revisions_by_title(title).last }
       end
 
       def markup

data/lib/caramelize/input_wiki/{wikkawiki.rb → wikka_wiki.rb}

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'caramelize/database_connector'
 require 'caramelize/filters/add_newline_to_page_end'
 require 'caramelize/filters/camel_case_to_wiki_links'
 require 'caramelize/filters/wikka_to_markdown'
@@ -17,21 +16,18 @@ module Caramelize
       def initialize(options = {})
         super(options)
         @options[:markup] = :wikka
-        @options[:filters] << Caramelize::AddNewlineOnPageEnd
-        @options[:filters] << Caramelize::Wikka2Markdown
-        @options[:filters] << Caramelize::CamelCaseToWikiLinks
+        @options[:filters] << ::Caramelize::AddNewlineToPageEnd
+        @options[:filters] << ::Caramelize::WikkaToMarkdown
+        @options[:filters] << ::Caramelize::CamelCaseToWikiLinks
       end
 
       # after calling this action, titles and @revisions are expected to be filled
       def read_pages
         pages.each do |row|
           titles << row['tag']
-          page = Page.new(build_properties(row))
-          revisions << page
+          revisions << Page.new(build_properties(row))
         end
         titles.uniq!
-        # revisions.sort! { |a,b| a.time <=> b.time }
-
         revisions
       end
 
@@ -60,7 +56,7 @@ module Caramelize
         @pages ||= database.query(pages_query)
       end
 
-      def build_properties(row) # rubocop:todo Metrics/MethodLength
+      def build_properties(row)
         author = authors[row['user']]
         {
           id: row['id'],

data/lib/caramelize/output_wiki/gollum.rb

@@ -38,7 +38,7 @@ module Caramelize
       def commit_history(revisions, options = {}, &block)
         revisions.each_with_index do |page, index|
           # call debug output from outside
-          block.call(page, index) if block_given?
+          yield(page, index) if block
           commit_revision(page, options.fetch(:markup, :markdown))
         end
       end

data/lib/caramelize/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Caramelize
-  VERSION = '1.2.1'
+  VERSION = '1.3.0'
 end

data/lib/caramelize.rb

@@ -2,18 +2,20 @@
 
 require 'caramelize/version'
 require 'caramelize/page'
-require 'caramelize/content_transferer'
-require 'caramelize/filter_processor'
 require 'caramelize/database_connector'
+require 'caramelize/filters/add_newline_to_page_end'
+require 'caramelize/filter_processor'
+require 'caramelize/input_wiki/wiki'
+require 'caramelize/input_wiki/media_wiki'
+require 'caramelize/input_wiki/redmine_wiki'
+require 'caramelize/input_wiki/wikka_wiki'
+require 'caramelize/content_transferer'
 require 'caramelize/health_check'
 require 'caramelize/health_checks/home_page_check'
 require 'caramelize/health_checks/orphaned_pages_check'
 require 'caramelize/health_checks/page'
 require 'caramelize/output_wiki/gollum'
 require 'caramelize/services/page_builder'
-require 'caramelize/input_wiki/wiki'
-require 'caramelize/input_wiki/redmine_wiki'
-require 'caramelize/input_wiki/wikkawiki'
 
 Encoding.default_external = Encoding::UTF_8
 Encoding.default_internal = Encoding::UTF_8

data/samples/media_wiki.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Connecting to the MediaWiki Docker setup  at https://hub.docker.com/_/mediawiki/
+
+def input_wiki
+  options = {
+    host: '0.0.0.0',
+    username: 'wikiuser',
+    password: 'example',
+    database: 'my_wiki',
+    port: 63_645
+  }
+  Caramelize::InputWiki::MediaWiki.new(options)
+end

data/spec/lib/caramelize/filters/add_newline_to_page_end_spec.rb

@@ -2,9 +2,7 @@
 
 require 'spec_helper'
 
-# rubocop:todo RSpec/SpecFilePathFormat
-describe Caramelize::Filter::AddNewlineToPageEnd do # rubocop:todo RSpec/FilePath, RSpec/SpecFilePathFormat
-  # rubocop:enable RSpec/SpecFilePathFormat
+describe Caramelize::AddNewlineToPageEnd do
   describe '#run' do
     subject(:run) { filter.run }
 

data/spec/lib/caramelize/filters/mediawiki_to_markdown_spec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Caramelize::MediawikiToMarkdown do
+  let(:filter) { described_class.new(body) }
+
+  describe '#run' do
+    subject { filter.run }
+
+    # the conversions are performed through pandoc-ruby.
+    # This is a smoke test to see if it's called at all,
+    # but not testing every single option.
+
+    context 'when headline h1' do
+      let(:body) { "= Headline =\n" }
+
+      it { is_expected.to eq "# Headline\n" }
+    end
+
+    context 'when headline h2' do
+      let(:body) { "== Headline ==\n" }
+
+      it { is_expected.to eq "## Headline\n" }
+    end
+  end
+end

data/spec/lib/caramelize/filters/{wikka2markdown_spec.rb → wikka_to_markdown_spec.rb}

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 # rubocop:todo RSpec/SpecFilePathFormat
-describe Caramelize::Wikka2Markdown do # rubocop:todo RSpec/FilePath, RSpec/SpecFilePathFormat
+describe Caramelize::WikkaToMarkdown do # rubocop:todo RSpec/FilePath, RSpec/SpecFilePathFormat
   # rubocop:enable RSpec/SpecFilePathFormat
   let(:filter) { described_class.new(body) }
 
@@ -71,37 +71,19 @@ describe Caramelize::Wikka2Markdown do # rubocop:todo RSpec/FilePath, RSpec/Spec
     end
 
     context 'when unordered list entry' do
-      context 'with tab based' do
+      context 'with tabs' do
         let(:body) { "\t-unordered list entry" }
 
         it { is_expected.to eq '- unordered list entry' }
       end
 
-      context 'with tabs' do
+      context 'with tilde' do
         let(:body) { '~-unordered list entry' }
 
         it { is_expected.to eq '- unordered list entry' }
       end
 
       context 'with spaces' do
-        let(:body) { '    -unordered list entry' }
-
-        it { is_expected.to eq '- unordered list entry' }
-      end
-
-      context 'with tab based with space' do
-        let(:body) { "\t- unordered list entry" }
-
-        it { is_expected.to eq '- unordered list entry' }
-      end
-
-      context 'with another tab based with space' do
-        let(:body) { '~- unordered list entry' }
-
-        it { is_expected.to eq '- unordered list entry' }
-      end
-
-      context 'with space based with space' do
         let(:body) { '    - unordered list entry' }
 
         it { is_expected.to eq '- unordered list entry' }
@@ -129,7 +111,7 @@ describe Caramelize::Wikka2Markdown do # rubocop:todo RSpec/FilePath, RSpec/Spec
         it { is_expected.to eq '[[LemmaLemma]]' }
       end
 
-      context 'with only url sklfs' do
+      context 'with only wikilink' do
         let(:body) { "\n        [[ComunitySiteIdeas]]        \n" }
 
         it { is_expected.to eq "\n        [[ComunitySiteIdeas]]        \n" }
@@ -261,5 +243,31 @@ describe Caramelize::Wikka2Markdown do # rubocop:todo RSpec/FilePath, RSpec/Spec
 
       it { is_expected.to eq expected_result }
     end
+
+    context 'when image' do
+      context 'with link' do
+        let(:body) { '{{image class="center" alt="DVD logo" title="An image link" url="images/dvdvideo.gif" link="RecentChanges"}}' }
+
+        it { is_expected.to eq '[[<img src="images/dvdvideo.gif" alt="DVD logo">|RecentChanges]]' }
+      end
+
+      context 'with alt and with title' do
+        let(:body) { '{{image class="center" alt="DVD logo" title="An image link" url="images/dvdvideo.gif"}}' }
+
+        it { is_expected.to eq '![DVD logo](images/dvdvideo.gif)' }
+      end
+
+      context 'without alt and with title' do
+        let(:body) { '{{image class="center" title="An image link" url="images/dvdvideo.gif"}}' }
+
+        it { is_expected.to eq '![](images/dvdvideo.gif)' }
+      end
+
+      context 'without alt and without title' do
+        let(:body) { '{{image class="center" url="images/dvdvideo.gif"}}' }
+
+        it { is_expected.to eq '![](images/dvdvideo.gif)' }
+      end
+    end
   end
 end

data/spec/lib/caramelize/input_wiki/media_wiki_spec.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Caramelize::InputWiki::MediaWiki do
+  subject(:wiki) { described_class.new }
+
+  describe '#filters' do
+    it 'has 2 filters' do
+      expect(wiki.filters.count).to be(2)
+    end
+  end
+end

data/spec/lib/caramelize/input_wiki/wikka_wiki_spec.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Caramelize::InputWiki::WikkaWiki do
+  subject(:wiki) { described_class.new }
+
+  describe '#filters' do
+    it 'has 3 filters' do
+      expect(wiki.filters.count).to be(3)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: caramelize
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Daniel Senff
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-10 00:00:00.000000000 Z
+date: 2024-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: commander
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: paru
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
@@ -78,6 +92,7 @@ files:
 - ".github/workflows/main.yml"
 - ".gitignore"
 - ".rubocop.yml"
+- ".rubocop_todo.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -93,6 +108,7 @@ files:
 - lib/caramelize/filter_processor.rb
 - lib/caramelize/filters/add_newline_to_page_end.rb
 - lib/caramelize/filters/camel_case_to_wiki_links.rb
+- lib/caramelize/filters/mediawiki_to_markdown.rb
 - lib/caramelize/filters/remove_table_tab_line_endings.rb
 - lib/caramelize/filters/swap_wiki_links.rb
 - lib/caramelize/filters/wikka_to_markdown.rb
@@ -100,13 +116,15 @@ files:
 - lib/caramelize/health_checks/home_page_check.rb
 - lib/caramelize/health_checks/orphaned_pages_check.rb
 - lib/caramelize/health_checks/page.rb
+- lib/caramelize/input_wiki/media_wiki.rb
 - lib/caramelize/input_wiki/redmine_wiki.rb
 - lib/caramelize/input_wiki/wiki.rb
-- lib/caramelize/input_wiki/wikkawiki.rb
+- lib/caramelize/input_wiki/wikka_wiki.rb
 - lib/caramelize/output_wiki/gollum.rb
 - lib/caramelize/page.rb
 - lib/caramelize/services/page_builder.rb
 - lib/caramelize/version.rb
+- samples/media_wiki.rb
 - spec/fixtures/markup/swap-links-input.textile
 - spec/fixtures/markup/swap-links-output.textile
 - spec/fixtures/markup/table-tab-line-endings-input.textile
@@ -115,10 +133,13 @@ files:
 - spec/lib/caramelize/filter_processor_spec.rb
 - spec/lib/caramelize/filters/add_newline_to_page_end_spec.rb
 - spec/lib/caramelize/filters/camel_case_to_wiki_links_spec.rb
+- spec/lib/caramelize/filters/mediawiki_to_markdown_spec.rb
 - spec/lib/caramelize/filters/remove_table_tab_line_endings_spec.rb
 - spec/lib/caramelize/filters/swap_wiki_links_spec.rb
-- spec/lib/caramelize/filters/wikka2markdown_spec.rb
+- spec/lib/caramelize/filters/wikka_to_markdown_spec.rb
+- spec/lib/caramelize/input_wiki/media_wiki_spec.rb
 - spec/lib/caramelize/input_wiki/wiki_spec.rb
+- spec/lib/caramelize/input_wiki/wikka_wiki_spec.rb
 - spec/lib/caramelize/output_wiki/gollum_spec.rb
 - spec/lib/caramelize/page_spec.rb
 - spec/lib/caramelize/services/page_builder_spec.rb