elexis-wiki-interface 0.4.8 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dcd4309470e3954cecd1e5d35c7a75efa7a453be
-  data.tar.gz: db7566d5c5ba624c72e3b08a084cf4f7e4b45091
+  metadata.gz: 4ce49a15cb845a0cdfc0877279106c0c3f07ac9a
+  data.tar.gz: f18d578327aa1ddf483de7036b12747fd37067fd
 SHA512:
-  metadata.gz: 7981f2ce10eb3c5be76ba97c47764f53623a1f083952a123accaeb9e2729f68e70ae261b45e0e55d873419145e2c861bbee2dd4eba258cd0de4c08e6b621a0db
-  data.tar.gz: 8290acb38b2b5543cf6d467550bc7f02a4be7dd9aa14b9bfeaa6df5fa61eddfc4d530511361ea87a2880a14008b4ff6adcc20c02ceb42aaf40d7d4c3f41605f8
+  metadata.gz: b97ac0b5d7affd17837da3ed44058c56b1bcc459b6fbb6351b3ea0509af483bdc722c45588bcb0edab24d0ac8995cd9b030b1c0d3c6f01cdb7467144c92d1261
+  data.tar.gz: bbea945c3ac300dd6adfa5e0f798486b3da44ed90b902ba65fc28c09ef9a6d39cde5f99825aff686cb815c5dc77606a7976cea55c8972d9495cf2b572378a418

data/Gemfile

@@ -5,4 +5,6 @@ gemspec
 
 
 gem 'eclipse-plugin' # , :git => 'https://github.com/ngiger/eclipse-plugin'
-# gem 'pry-nav'
+group :debugger do
+  gem 'pry-byebug'
+end

data/Gemfile.lock

@@ -1,15 +1,20 @@
 PATH
   remote: .
   specs:
-    elexis-wiki-interface (0.4.8)
+    elexis-wiki-interface (0.5.0)
       eclipse-plugin (>= 0.1)
       mediawiki-gateway
+      mediawiki_api
       rubyzip (< 1.0.0)
       unicode
 
 GEM
   remote: https://rubygems.org/
   specs:
+    byebug (5.0.0)
+      columnize (= 0.9.0)
+    coderay (1.1.0)
+    columnize (0.9.0)
     diff-lcs (1.2.5)
     docile (1.1.5)
     domain_name (0.5.24)
@@ -17,16 +22,33 @@ GEM
     eclipse-plugin (0.3)
       nokogiri
       rubyzip (< 1.0.0)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
+    faraday-cookie_jar (0.0.6)
+      faraday (>= 0.7.4)
+      http-cookie (~> 1.0.0)
     http-cookie (1.0.2)
       domain_name (~> 0.5)
     json (1.8.3)
     mediawiki-gateway (1.0.0)
       rest-client (~> 1.7)
+    mediawiki_api (0.5.0)
+      faraday (~> 0.9, >= 0.9.0)
+      faraday-cookie_jar (~> 0.0, >= 0.0.6)
+    method_source (0.8.2)
     mime-types (2.6.1)
     mini_portile (0.6.2)
+    multipart-post (2.0.0)
     netrc (0.10.3)
     nokogiri (1.6.6.2)
       mini_portile (~> 0.6.0)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-byebug (3.2.0)
+      byebug (~> 5.0)
+      pry (~> 0.10)
     rake (10.4.2)
     rest-client (1.8.0)
       http-cookie (>= 1.0.2, < 2.0)
@@ -51,6 +73,7 @@ GEM
       json (~> 1.8)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
+    slop (3.6.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.1)
@@ -63,6 +86,7 @@ DEPENDENCIES
   bundler (~> 1.10)
   eclipse-plugin
   elexis-wiki-interface!
+  pry-byebug
   rake (~> 10.0)
   rspec
   simplecov

data/README.textile

@@ -12,7 +12,7 @@ Copyright: 2014 (c) by Niklaus Giger <niklaus.giger@member.fsf.org>
 
 h2. Installation
 
-Ensure that you use Ruby 1.9.2 or higher (mediawiki-gateway does not support Ruby 1.8.7).
+Ensure that you use Ruby 1.9.2 or higher. Mediawiki 1.19 does not work. Mediawiki 1.25.2 works. Don't know how the versions between score.
 
 Add this line to your application's Gemfile:
 
@@ -79,9 +79,8 @@ Also we don't use the tag Datei, as this one is not supported by the Mylin Wikit
 ** a perspective perspective.name with a localized name localized_name has a page  P_<localized_name>
 ** an image file must be lowercase (files which differ only by the case will be deleted)
 ** use syntax @[[File:@, not [[Datei: or @[[Image:@
-** png files should start with their plugin/feature-name followed by '/' by a name or a globally unique name
+** png files should start with a name derived from their symbolic name, eg, molemax followed by '-' by a name or a globally unique name
 ** png files resides inside the 'doc' sub-directory of each plugin/feature
-** inside the doc directoy we create a logical link of the plugin|feature name to the directory. This allows us to edit and preview the mediawiki files using the Mylin Wikitext extension of Eclipse.
 
 h3. Limitations/TODO
 

data/bin/wiki_pull

@@ -1,6 +1,9 @@
 #!/bin/env ruby
-require 'elexis/wiki/interface/workspace'
-
-workspace =  Elexis::Wiki::Interface::Workspace.new(Dir.pwd)
+require 'elexis/wiki/workspace'
+workspace =  Elexis::Wiki::Workspace.new(Dir.pwd)
 workspace.pull
+
+images = Elexis::Wiki::Images.new(Dir.pwd)
+images.determine_cleanup
+images.execute_cleanup
 workspace.show_missing(true)

data/bin/wiki_pull_and_commit

@@ -0,0 +1,38 @@
+#!/bin/env ruby
+
+require 'elexis/wiki/workspace'
+
+workspace =  Elexis::Wiki::Workspace.new(Dir.pwd)
+workspace.pull
+
+images = Elexis::Wiki::Images.new(Dir.pwd)
+images.determine_cleanup
+images.execute_cleanup(true) # adds changes via git add
+
+files = Dir.glob('doc_??') +
+    Dir.glob('*/doc/*.mediawiki', File::FNM_CASEFOLD) +
+    Dir.glob('*/doc/*.png', File::FNM_CASEFOLD)
+    Dir.glob('*/doc/*.jpg', File::FNM_CASEFOLD)
+    Dir.glob('*/doc/*.gif', File::FNM_CASEFOLD)
+(Dir.glob("doc_??/*") + Dir.glob('*/doc/*')).each{|f| files << f if File.symlink?(f) }
+files.each{
+  |file|
+  if /:/.match(File.basename(file))
+      puts "You may not add a file containg ':' or it will break git for Windows. Remove/rename #{file}"
+      exit 2
+  end
+}
+
+if (error_files = Dir.glob('**/*:*')) and error_files.size > 0
+  puts "You may not add a file containg ':' or it will break git for Windows. Remove/rename #{error_files}"
+  exit 3
+end
+
+if true
+  system("git status")
+  system("git commit --all -m '#{File.basename(__FILE__)}: #{Date.today.to_s} from #{workspace.if.wiki_url.sub(/api.php/i, '')}'")
+  system("git status")
+  system("git log -1")
+  # system("git push")
+end
+workspace.show_missing(true)

data/bin/wiki_push

@@ -1,5 +1,5 @@
 #!/bin/env ruby
-require 'elexis/wiki/interface/workspace'
-workspace =  Elexis::Wiki::Interface::Workspace.new(Dir.pwd)
+require 'elexis/wiki/workspace'
+workspace =  Elexis::Wiki::Workspace.new(Dir.pwd)
 workspace.push
 

data/elexis-wiki-interface.gemspec

@@ -1,11 +1,11 @@
 # coding: utf-8
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'elexis/wiki/interface/version'
+require 'elexis/wiki/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "elexis-wiki-interface"
-  spec.version       = Elexis::Wiki::Interface::VERSION
+  spec.version       = Elexis::Wiki::VERSION
   spec.authors       = ["Niklaus Giger"]
   spec.email         = ["niklaus.giger@member.fsf.org"]
   spec.summary       = "Interface between elexis source and wiki"
@@ -19,7 +19,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.add_dependency 'rubyzip', '< 1.0.0'
   spec.add_dependency 'unicode'
-  spec.add_dependency 'mediawiki-gateway'
+  spec.add_dependency 'mediawiki-gateway'     # needed to get pages and infos about pages
+  spec.add_dependency 'mediawiki_api'         # needed to push images
   spec.add_dependency 'eclipse-plugin', '>= 0.1'
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"

data/history.textile

@@ -0,0 +1,14 @@
+h3. Version 0.5.0 (2015-09-18)
+
+* Major cleanup, much more spec tests
+* Refactored accessing the Wiki into the new class Interface
+* Using direct json, mediawiki-gateway and mediawiki_api as I found no good gem, which does allow all stuff
+* Requires newer mediawiki (actuall 1.25.2, 1.19 does not work. Unknown how it behaves between)
+* Recognize duplicates in pictures
+* Forces renaming of old convention with '/' or ':' in picture names
+* Added pictures.csv|yml to store information about the seen picture (duplicates, etc)
+
+h3. Version 0.4.8 (2015-08-15)
+
+* Fails miserably uploading to Mediawiki 1.19
+

data/lib/elexis/wiki/images.rb

@@ -0,0 +1,338 @@
+#encoding: utf-8
+
+require 'eclipse/plugin'
+require 'media_wiki'
+require 'mediawiki_api'
+require 'elexis/wiki/images'
+require 'fileutils'
+require 'open-uri'
+require 'time'
+require 'yaml'
+require 'csv'
+
+module Elexis
+  module Wiki
+    ImagePattern = /(\[File:|Datei:|\[Image:)([ \w\.\:\/]*)/i
+
+    #
+    # This helper class collect information about all images used in
+    # subdirectories doc ending with mediawiki and stores the results
+    # in a pictures.yml and a pictures.csv file
+    #
+    # Several helper allow showing unused pictures, identifying
+    # files with same name, but different content and pictures use
+    # in several sub-directories
+    #
+    class Images
+
+      private
+        def write_yml_and_csv
+          File.open(@yml, 'w+') {|f| f.puts @pictures.to_yaml}
+          column_header = ["sha256","name","project","path"]
+          CSV.open(csv, 'w+',
+                   :write_headers=> true,
+                   :headers => column_header
+                  ) do |csv|
+                csv << ["# Generated by: #{File.basename(__FILE__)} at #{Time.now.utc}"]
+                @pictures.each{ |pict| csv << pict.values.to_a  }
+          end
+        end
+        def initialize_database
+          savedDir = Dir.pwd
+          Dir.chdir(@rootDir)
+          @docs = Dir.glob('**/doc').reject{|x| x.match(/(^vendor|\/vendor)\//) } # doc_fr is mostly a duplicate of doc_de
+          @docs.each{
+            |doc_dir|
+            pngs = Dir.glob("#{doc_dir}/**/*.{png,jpg}")
+            puts "#{doc_dir} has #{pngs.size} png" if $VERBOSE
+            next if pngs.size == 0
+            pngs.each {
+                      |png|
+                      sha256 = Digest::SHA256.hexdigest(IO.read(png))
+                      pict = { :sha256 => sha256,
+                               :name => File.basename(png),
+                               :project => File.basename(File.dirname(File.dirname(png))),
+                               :path => png}
+                      @pictures << pict
+                      }
+            # break if pngs.size > 1
+          }
+          @sha_2_png = {}
+          @pictures.each {
+            |picture|
+            if @sha_2_png[picture[:sha256]]
+              @sha_2_png[picture[:sha256]] << picture
+            else
+              @sha_2_png[picture[:sha256]] = [picture]
+            end
+          }
+          @duplicates        = @sha_2_png.find_all{|sha, list| list.size > 1}
+          write_yml_and_csv
+        ensure
+          Dir.chdir(savedDir)
+        end
+
+        def verify_best_name(picture)
+          if picture[:best_name]
+            wrong_best_name = @pictures.find_all{|x| x[:best_name] == picture[:best_name] and x[:sha256] != picture[:sha256]}
+            if wrong_best_name.size > 0
+              puts "   #{wrong_best_name.size} entries #{wrong_best_name}" #if $VERBOSE
+              @wrong_best_name << [picture[:path], wrong_best_name.collect{|x| x[:path]} ]
+              new_name = get_name_with_project(picture[:path])
+              if picture[:best_name] != new_name
+                puts "new_best_name 2 is #{new_name} for #{picture[:path]}" #if $VERBOSE
+                picture[:rule] += " name get_name_with_project"
+                picture[:best_name] = new_name
+              else
+                @@uniq_id ||= 0
+                new_name = '27'
+                begin
+                  @@uniq_id += 1
+                  new_name = File.dirname(picture[:path]) + "/#{@@uniq_id}_" + picture[:name]
+                end until File.exist?(new_name).eql?(false)
+                puts "new_best_name 3 is #{new_name} for #{picture[:path]}" # if $VERBOSE
+                picture[:rule] += " third #{new_name}"
+                picture[:best_name] = File.basename(new_name)
+              end
+            end
+          end
+        end
+
+        def set_best_name(picture)
+          debug = $VERBOSE
+          path = picture[:path]
+          sha256 = Digest::SHA256.hexdigest(IO.read(path))
+          same_sha256 = @pictures.find_all{|x| x[:sha256] == sha256}
+          same_name   = @pictures.find_all{|x| x[:name] == picture[:name]}
+          nrSha256 = same_name.collect{|c| get_small_name(c[:sha256])}.uniq
+          if same_sha256.size == 1 and nrSha256.size == 1
+            puts "new_best_name 3 @short_and_sha_okay #{sha256} for #{path}" if debug
+            picture[:rule] = 'short_and_sha_okay nr_single_sha'
+            @short_and_sha_okay << path
+            @nr_single_sha << same_sha256
+            picture[:best_name] = get_small_name(path)
+            return
+          else
+            msg = " nrSha #{sha256.size} #{sha256}"
+            if same_sha256.collect{|c| c[:name]}.uniq.size == 1 and nrSha256.size == 1
+              puts "@short_and_sha_okay #{path} sha256 #{sha256} for #{same_sha256.size} files" if debug
+              picture[:rule] = 'short_and_sha_okay'
+              @short_and_sha_okay << path
+              return
+            end
+            found = same_sha256.collect{|c| c[:name].downcase}.uniq
+            if found.size == 1  and nrSha256.size == 1
+              puts "new_best_name 4 case_sensitives #{found}" if debug
+              picture[:rule] = 'case_sensitives'
+              picture[:best_name] = get_small_name(path)
+              return
+            end
+            multiple = same_sha256.collect{|c| c[:name]}.uniq
+            to_reduce = same_sha256.collect{|c| get_small_name(c[:name])}.uniq
+            if nrSha256.size == 1
+              @to_short_names[path] = to_reduce
+              puts "new_best_name 5 to_short_names #{multiple} =>  #{to_reduce} nrSha #{sha256.size} #{sha256}" if debug
+              picture[:rule] = 'to_short_names'
+              picture[:best_name] = get_small_name(path)
+              return
+            end
+            puts "new_best_name  6 multiple #{multiple} to_reduce #{to_reduce} #{get_name_with_project(picture[:path])}" if debug
+            @multiple_names[path] = same_sha256
+            picture[:best_name] =  get_name_with_project(picture[:path])
+            picture[:rule] = 'multiple_names'
+            return
+          end
+        end
+      public
+      attr_reader :rootDir, :docDir, :extension, :yml, :csv, :pictures, :sha_2_png,
+          :multiple_names, :to_short_names, :nr_single_sha,
+          :short_and_sha_okay,  :wrong_best_name, :new_best_name,
+          :duplicates, :dup_non_identical, :actions
+
+      def initialize(rootDir = Dir.pwd, docDir = 'doc', extension = '.mediawiki')
+        @rootDir            = rootDir
+        @docDir             = docDir
+        @extension          = extension
+        @add_changes_to_git = false
+        @pictures           = []
+        @multiple_names     = {}
+        @to_short_names     = {}
+        @nr_single_sha      = []
+        @short_and_sha_okay = []
+        @wrong_best_name    = []
+        @new_best_name      = {}
+        @yml                = File.expand_path(File.join(@rootDir, 'pictures.yml'))
+        @csv                = File.expand_path(File.join(@rootDir, 'pictures.csv'))
+        initialize_database
+      end
+
+      def get_name_with_project(path)
+        dir = File.dirname(path)
+        found = dir.sub(/(\.feature|[._]test.*|)\/doc/i, '').sub(/\.v\d$/, '')
+        result = (found.split('.')[-1] + '-' + File.basename(path)).downcase.gsub(':', '-')
+        result
+      end
+
+      def get_small_name(path)
+        ext =  File.extname(path)
+        to_consider = File.basename(path, ext).gsub(':','-')
+        if to_consider.index('.')
+          part = to_consider.split('.')[-1].downcase
+          if  part.split('_').size > 1
+            result = part.split('_')[1..-1].join('_')
+          else
+            result = part
+          end
+        else
+          result = to_consider
+        end
+        (result + ext).downcase
+      end
+
+      def determine_cleanup
+        savedDir = Dir.pwd
+        Dir.chdir(@rootDir)
+        @pictures.each{ |picture| set_best_name(picture) }
+        @pictures.each{ |picture| verify_best_name(picture) }
+        @wrong_best_name = []
+        @pictures.each{ |picture| verify_best_name(picture) }
+        @dup_non_identical = @pictures.collect{ |p| p[:path] if p[:rule] =~ /multiple/}.compact
+        @pictures.find{ |outer| @pictures.find_all{ |inner| inner[:name].eql?(outer[:name]) and not inner[:sha256].eql?(outer[:sha256])}.size > 1 }
+        @new_best_name     = @pictures.find_all{ |picture| picture[:best_name] and not picture[:best_name].downcase.eql?(picture[:name].downcase) }
+        write_yml_and_csv
+      ensure
+        Dir.chdir(savedDir)
+      end
+
+      def remove_obsolete_symlinks
+        @docs.each do
+          |docDir|
+          Dir.chdir(File.join(@rootDir, docDir))
+          project =  File.basename(File.dirname(docDir))
+          tries = [project, project.capitalize, project.sub('.feature', ''), project.sub('.feature', '').capitalize]
+          files = []
+          tries.each{|try| files << try if File.symlink?(try) }
+          files.compact.each do
+            |symlink|
+            if symlink and File.symlink?(symlink)
+                cmd = "rm #{symlink}"
+                @actions << cmd
+                FileUtils.rm_f(symlink, :verbose => true)
+                run_git_cmd("git rm -f --ignore-unmatch #{symlink}") if @add_changes_to_git
+            end
+          end
+        end
+      end
+
+      def remove_files_with_case_sensitive_changes(file_correct_case)
+        files = Dir.glob(file_correct_case + '*', File::FNM_CASEFOLD)
+        return if files.size == 1
+        files.each{
+          |file|
+            next if File.basename(file).eql?(File.basename(file_correct_case))
+            if @add_changes_to_git
+              run_git_cmd("git rm -f #{file}")
+            else
+              FileUtils.rm_f(file)
+            end
+        }
+      end
+
+      def run_git_cmd(git_cmd)
+        return unless @add_changes_to_git
+        res = system(git_cmd)
+        unless res
+          binding.pry
+          raise "Running #{git_cmd} failed"
+        end
+        @actions  << git_cmd
+      end
+
+      def execute_cleanup(add_changes_to_git=false)
+        @add_changes_to_git = add_changes_to_git
+        @actions ||= []
+        savedDir = Dir.pwd
+        remove_obsolete_symlinks
+        Dir.chdir(@rootDir)
+        cmds = []
+        @new_best_name.each{
+          |picture|
+          dir_name = File.dirname(picture[:path])
+          Dir.chdir(File.join(@rootDir, dir_name))
+          Dir.glob("*#{extension}").each  do
+            |wiki_file|
+          old_image_name = picture[:name]
+          new_image_name = picture[:best_name]
+          if old_image_name.index('-')
+            puts "Skipping mv #{old_image_name} #{new_image_name}"
+            next
+          end
+          cmd = "change_image_name_in_mediawiki #{wiki_file} #{picture[:name]} #{picture[:best_name]}"
+          # cmd = 'change_image_name_in_mediawiki test.mediawiki ch.elexis.icpc_icpc1.png icpc1.png'
+          change_image_name_in_mediawiki wiki_file, old_image_name, new_image_name
+          @actions  << cmd
+          oldFiles = Dir.glob(old_image_name, File::FNM_CASEFOLD)
+          oldFiles.each{ |old_file|
+                        if @add_changes_to_git
+                          run_git_cmd("git rm -f --ignore-unmatch #{old_file}") if File.exist?(old_file)
+                        else
+                          FileUtils.rm_f(old_file, :verbose => true)
+                          @actions  << "rm -f #{old_file}"
+                        end
+                       }
+        end
+        }
+        Dir.chdir(@rootDir)
+        # Add all pictures which were downloaded and still have the same name
+        @pictures.each { |picture| run_git_cmd("git add -f #{picture[:path]}") if File.exist?(picture[:path]) }
+        @actions.uniq!
+        puts @actions.join("\n")
+      ensure
+        Dir.chdir(savedDir)
+      end
+
+      # Helper for scripts
+      # We assume that the referenced new_image_name is living in the same directory
+      # as the mediawiki_file
+      def change_image_name_in_mediawiki(mediawiki_file, old_image_name, new_image_name)
+        if new_image_name.downcase != File.basename(new_image_name).downcase
+          raise "new_image_name #{new_image_name} may not contain a directory path"
+        end
+        if @add_changes_to_git
+          run_git_cmd("git add #{old_image_name} && git mv #{old_image_name} #{new_image_name}")
+        elsif @actions
+          @actions << "mv #{old_image_name} #{new_image_name}"
+          FileUtils.mv(old_image_name, new_image_name, :verbose => true)
+        end unless File.exists?(new_image_name)
+        puts "change_image_name_in_mediawiki #{Dir.pwd}/#{mediawiki_file}: #{old_image_name.inspect} => #{new_image_name.inspect}" # if $VERBOSE
+        newLines = []
+        lines = IO.readlines(mediawiki_file)
+        lines.each{
+          |line|
+          unless m = ImagePattern.match(line)
+            newLines << line
+          else
+            unless old_image_name.downcase.eql?(m[2].downcase) or  /\/#{old_image_name}/i.match(m[2])
+              puts "change_image_name_in_mediawiki #{__LINE__}  skip #{m}" if $VERBOSE
+              newLines << line
+            else
+              dirName = File.dirname(mediawiki_file)
+              simpleName = File.join(dirName, File.basename(new_image_name))
+              if files = Dir.glob(simpleName, File::FNM_CASEFOLD) and files.size >= 1
+              # if files = (Dir.glob(simpleName, File::FNM_CASEFOLD) +Dir.glob(old_image_name, File::FNM_CASEFOLD)) and files.size >= 1
+                new_line = line.sub(m[2], new_image_name)
+                puts "change_image_name_in_mediawiki #{__LINE__}  #{line} aus #{m[2]} mit #{new_image_name}" if $VERBOSE
+                newLines << new_line
+              else
+                msg =  "#{__LINE__} Could not find image for #{m[0]} searched for #{simpleName} in #{Dir.pwd}. files are #{files}"
+                raise msg
+              end
+            end
+          end
+        }
+        File.open(mediawiki_file, "w") {|f| f.write newLines.join.gsub(/\[\[Datei:|\[\[Image:/i, '[[File:')}
+        run_git_cmd("git add #{mediawiki_file}")
+      end
+    end
+  end
+end