cartocss_helper 1.2.1 → 4.0.0

data/lib/cartocss_helper/util/logger.rb

@@ -0,0 +1,30 @@
+require 'logger'
+
+module Log
+  @@_logger_ = Logger.new(STDOUT)
+
+  def self.logger
+    @@_logger_
+  end
+
+  def self.info(message = '')
+    logger.info(message)
+  end
+
+  def self.warn(message = '')
+    logger.warn(message)
+  end
+
+  @@_logger_.formatter = proc do |_severity, _datetime, _progname, msg|
+    fileLine = ''
+    caller.each do |clr|
+      unless /\/logger.rb:/ =~ clr
+        fileLine = clr
+        break
+      end
+    end
+    fileLine = fileLine.split(':in `', 2)[0]
+    fileLine.sub!(/:(\d)/, '(\1')
+    "#{fileLine}) : #{msg}\n"
+  end
+end

data/lib/cartocss_helper/util/rest-client_wrapper.rb

@@ -0,0 +1,53 @@
+require 'rest-client'
+require_relative 'generic_downloader.rb'
+
+class RestClientWrapper
+  def initialize
+    @last_url_fetched = nil
+  end
+
+  def fetch_data_from_url(url, request_timeout)
+    wait if url == @last_url_fetched
+    @last_url_fetched = url
+    return execute_fetch_data_from_url(url, request_timeout)
+  # http://www.rubydoc.info/gems/rest-client/1.8.0/RestClient/Exception
+  rescue RestClient::RequestTimeout => e
+    raise RequestTimeout, e.to_s
+  rescue RestClient::ExceptionWithResponse => e
+    raise_exception_about_returned_response(e)
+  rescue RestClient::MaxRedirectsReached, RestClient::SSLCertificateNotVerified, RestClient::ServerBrokeConnection, SocketError, URI::InvalidURIError
+    raise ExceptionWithoutResponse.new(e), e.to_s
+  rescue ArgumentError => e
+    raise_issue_359_exception(e)
+  rescue => e
+    puts 'unhandled exception! It is a clear bug!'
+    puts "<#{e.class} error happened>"
+    raise e, e.to_s
+  end
+
+  def execute_fetch_data_from_url(url, request_timeout)
+    data = RestClient::Request.execute(method: :get, url: url, timeout: request_timeout)
+    # see https://github.com/rest-client/rest-client/issues/370, will be fixed in 2.0
+    # published versions listed on https://rubygems.org/gems/rest-client/versions/
+    return String.new(data)
+  end
+
+  def raise_issue_359_exception(e)
+    raise BuggyRestClient, "ArgumentError from rest-client, most likely caused by https://github.com/rest-client/rest-client/issues/359 (requesting GBs of data) [#{e}]"
+  end
+
+  def raise_exception_about_returned_response(e)
+    failure = ExceptionWithResponse.new(e)
+    failure.http_code = e.http_code
+    failure.response = e.response
+    raise failure, e.to_s
+  end
+
+  def wait
+    progressbar = ProgressBar.create
+    100.times do
+      sleep 4
+      progressbar.increment
+    end
+  end
+end

data/lib/cartocss_helper/util/systemhelper.rb

@@ -0,0 +1,55 @@
+class FailedCommandException < StandardError
+end
+
+module SystemHelper
+  def check_error_code(command, stderr, status, allowed_return_codes)
+    returned = status.exitstatus
+    return if status.success?
+    return if allowed_return_codes.include?(returned)
+    explanation = "failed command #{command} due to error code #{returned}. stderr was #{stderr}"
+    raise FailedCommandException.new(explanation)
+  end
+
+  def check_stderr(command, stderr, status, ignore_stderr_presence)
+    return if ignore_stderr_presence
+    return if stderr == ''
+    returned = status.exitstatus
+    explanation = "failed command #{command} due to < #{stderr}> on stderr. return code was #{returned}"
+    raise FailedCommandException.new(explanation)
+  end
+
+  def execute_command(command, debug = false, allowed_return_codes: [], ignore_stderr_presence: false)
+    puts command if debug
+    stdout, stderr, status = Open3.capture3(command)
+
+    check_error_code(command, stderr, status, allowed_return_codes)
+    check_stderr(command, stderr, status, ignore_stderr_presence)
+
+    return stderr + stdout
+  end
+
+  def self.not_enough_free_space
+    minimum_gb = 2
+    available_gb = get_available_space_for_cache_in_gb
+    if available_gb < minimum_gb
+      puts "get_available_space_for_cache_in_gb: #{available_gb}, minimum_gb: #{minimum_gb}"
+      return true
+    else
+      return false
+    end
+  end
+
+  def get_available_space_for_cache_in_gb
+    stat = Sys::Filesystem.stat(CartoCSSHelper::Configuration.get_path_to_folder_for_cache)
+    return stat.block_size * stat.blocks_available / 1024 / 1024 / 1024
+  end
+
+  def delete_file(filename, reason)
+    open(CartoCSSHelper::Configuration.get_path_to_folder_for_output + 'deleting_files_log.txt', 'a') do |file|
+      message = "deleting #{filename}, #{File.size(filename) / 1024 / 1024}MB - #{reason}"
+      puts message
+      file.puts(message)
+      File.delete(filename)
+    end
+  end
+end

data/lib/cartocss_helper/validator.rb

@@ -10,15 +10,32 @@ include CartoCSSHelper::Configuration
 module CartoCSSHelper
   module Validator
     def run_tests(git_branch)
-      run_expected_rendering_test(git_branch)
       run_dy_test(git_branch)
+      run_missing_name_test(git_branch)
+      run_expected_rendering_test(git_branch, true)
       run_closed_way_test(git_branch)
+      run_expected_rendering_test(git_branch)
+      run_unwanted_name_test(git_branch)
     end
 
-    def run_expected_rendering_test(git_branch)
+    def run_expected_rendering_test(git_branch, quick_and_more_prone_to_errors = false)
       Git.checkout git_branch
       puts 'unexpectedly rendered/unrendered keys:'
-      compare_expected_with_real_rendering
+      compare_expected_with_real_rendering(quick_and_more_prone_to_errors)
+      puts
+    end
+
+    def run_missing_name_test(git_branch)
+      Git.checkout git_branch
+      puts 'Check whatever names are displayed:'
+      run_global_check(:check_missing_names)
+      puts
+    end
+
+    def run_unwanted_name_test(git_branch)
+      Git.checkout git_branch
+      puts 'Check whatever names are displayed:'
+      run_global_check(:check_unwanted_names)
       puts
     end
 
@@ -36,137 +53,181 @@ module CartoCSSHelper
       puts
     end
 
-    def compare_expected_with_real_rendering
+    def compare_expected_with_real_rendering(quick_and_more_prone_to_errors = false)
       list_of_documented = CartoCSSHelper::Configuration.get_style_specific_data.list_of_documented_tags
       info = Info.new
-      list_of_rendered = info.get_render_state_of_tags
+      list_of_rendered = info.get_render_state_of_tags(quick_and_more_prone_to_errors)
 
+      puts 'ensure_that_tags_documented_and_rendered_have_the_same_state'
       ensure_that_tags_documented_and_rendered_have_the_same_state(list_of_documented, list_of_rendered)
+
+      puts 'ensure_that_all_rendered_tags_are_documented'
       ensure_that_all_rendered_tags_are_documented(list_of_documented, list_of_rendered)
+
+      puts 'ensure_that_all_documented_are_really_rendered'
       ensure_that_all_documented_are_really_rendered(list_of_documented, list_of_rendered)
     end
 
     def ensure_that_tags_documented_and_rendered_have_the_same_state(list_of_documented, list_of_rendered)
-      list_of_rendered.each { |tag_info|
+      list_of_rendered.each do |tag_info|
         compare_expected_with_tag_data list_of_documented, tag_info
-      }
+      end
     end
 
     def ensure_that_all_rendered_tags_are_documented(list_of_documented, list_of_rendered)
-      list_of_rendered.each { |tag_info|
-        if tag_info.state != :ignored
-          if !is_tag_documented(list_of_documented, tag_info.key, tag_info.value)
-            puts 'documented'
-            puts "\tmissing"
-            puts 'real'
-            print "\t"
-            tag_info.print
-            puts
-          end
-        end
-      }
+      list_of_rendered.each do |tag_info|
+        next if is_tag_documented(list_of_documented, tag_info.key, tag_info.value)
+        puts 'documented'
+        puts "\tmissing"
+        puts 'real'
+        print "\t"
+        tag_info.print
+        puts
+      end
     end
 
     def ensure_that_all_documented_are_really_rendered(list_of_documented, list_of_rendered)
-      list_of_documented.each { |documented|
-        if !is_tag_documented(list_of_rendered, documented.key, documented.value)
-          puts 'documented'
-          print "\t"
-          documented.print
-          puts 'real'
-          puts "\tmissing"
-          puts
+      list_of_documented.each do |documented|
+        if Info.get_expected_state(documented.key, documented.value) == :ignored
+          next
         end
-      }
+        next if is_tag_documented(list_of_rendered, documented.key, documented.value)
+        puts 'documented'
+        print "\t"
+        documented.print
+        puts 'real'
+        puts "\tmissing"
+        puts
+      end
     end
 
     def is_tag_documented(list, key, value)
-      list.each { |tag_info|
-        if key == tag_info.key
-          if value == tag_info.value
-            return true
-          end
-        end
-      }
+      list.each do |tag_info|
+        next unless key == tag_info.key
+        return true if value == tag_info.value
+      end
       return false
     end
 
     def compare_expected_with_tag_data(list_of_expected, tag_info)
-      list_of_expected.each { |expected|
-        if expected.key == tag_info.key
-          if expected.value == tag_info.value
-            if expected.equal? tag_info
-              puts 'expected'
-              print "\t"
-              expected.print
-              puts 'real'
-              print "\t"
-              tag_info.print
-              puts
-            end
-            return
-          end
+      list_of_expected.each do |expected|
+        next unless expected.key == tag_info.key
+        next unless expected.value == tag_info.value
+        if expected.equal? tag_info
+          puts 'expected'
+          print "\t"
+          expected.print
+          puts 'real'
+          print "\t"
+          tag_info.print
+          puts
         end
-      }
+        return
+      end
     end
 
     def run_global_check(function)
-      Heuristic.get_tags.each { |element|
-        (Configuration.get_max_z..Configuration.get_max_z).each { |zlevel| #get_min_z should be used - but as renderer is extremely slow this hack was used TODO
-          send(function, element[0], element[1], zlevel)
-        }
-      }
-    end
-
-    #TODO - what about composite tags?
-    def check_problems_with_closed_line(key, value, zlevel, on_water = false)
-      way = Scene.new({key => value}, zlevel, on_water, 'way')
-      closed_way = Scene.new({key => value}, zlevel, on_water, 'closed_way')
-      empty = Scene.new({}, zlevel, on_water, 'node')
+      Heuristic.get_tags.each do |element|
+        key = element[0]
+        value = element[1]
+        state = Info.get_expected_state(key, value)
+        tags = { key => value }
+        next if state == :ignore
+        if state == :composite
+          tags.merge!(Info.get_expected_composite(key, value))
+        end
+        (Configuration.get_max_z..Configuration.get_max_z).each do |zlevel| # get_min_z should be used - but as renderer is extremely slow this hack was used TODO
+          send(function, tags, zlevel)
+        end
+      end
+    end
+
+    def check_problems_with_closed_line(tags, zlevel, on_water = false)
+      way = Scene.new(tags, zlevel, on_water, 'way', true)
+      closed_way = Scene.new(tags, zlevel, on_water, 'closed_way', true)
+      empty = Scene.new({}, zlevel, on_water, 'node', true)
       if way.is_output_different(empty)
-        if !closed_way.is_output_different(empty)
-          puts key+'='+value
+        puts tags unless closed_way.is_output_different(empty)
+      end
+    end
+
+    def check_missing_names(tags, zlevel, interactive = false, on_water = false)
+      not_required = CartoCSSHelper::Configuration.get_style_specific_data.name_label_is_not_required
+      return if not_required.include?(tags)
+      ['node', 'closed_way', 'way'].each do |type|
+        next if not_required.include?(tags.merge({ type: type }))
+        unless is_object_displaying_anything_as_this_object_type tags, zlevel, on_water, type
+          # puts key+"="+value+" - not displayed as node on z#{zlevel}"
+          next
+        end
+        tags.delete('name') if tags['name'] != nil
+        unless is_object_displaying_name_as_this_object_type tags, 'a', zlevel, on_water, type
+          puts "#{tags} - label is missing on z#{zlevel} #{type}"
+          next
         end
       end
     end
 
-    def check_dy(key, value, zlevel, interactive=false, on_water=false)
-      if !is_object_displaying_anything key, value, zlevel, on_water
-        #puts key+"="+value+" - not displayed as node on z#{zlevel}"
+    def check_unwanted_names(tags, zlevel, interactive = false, on_water = false)
+      ['node', 'closed_way', 'way'].each do |type|
+        not_required = CartoCSSHelper::Configuration.get_style_specific_data.name_label_is_not_required
+        next unless not_required.include?(tags) || not_required.include?(tags.merge({ type: type }))
+        unless is_object_displaying_anything_as_this_object_type tags, zlevel, on_water, type
+          # puts key+"="+value+" - not displayed as node on z#{zlevel}"
+          next
+        end
+        tags.delete('name') if tags['name'] != nil
+        if is_object_displaying_name_as_this_object_type tags, 'a', zlevel, on_water, type
+          puts "#{tags} - label is unxpectedly displayed on z#{zlevel} #{type}"
+        end
+      end
+    end
+
+    def check_dy(tags, zlevel, interactive = false, on_water = false)
+      unless is_object_displaying_anything_as_node tags, zlevel, on_water
+        # puts key+"="+value+" - not displayed as node on z#{zlevel}"
         return
       end
-      if !is_object_displaying_name key, value, 'a', zlevel, on_water
-        #puts key+"="+value+" - label is missing on z#{zlevel} nodes"
+      unless is_object_displaying_name_as_node tags, 'a', zlevel, on_water
+        # puts key+"="+value+" - label is missing on z#{zlevel} nodes"
         return
       end
       test_name = 'ÉÉÉÉÉÉ ÉÉÉÉÉÉ ÉÉÉÉÉÉ'
-      while !is_object_displaying_name key, value, test_name, zlevel, on_water
-        puts key+'='+value+" - name is missing for name '#{test_name}' on z#{zlevel}"
+      until is_object_displaying_name_as_node tags, test_name, zlevel, on_water
+        puts "#{tags} - name is missing for name '#{test_name}' on z#{zlevel}"
         if interactive
           puts 'press enter'
           gets
         else
           return
         end
-        with_name = Scene.new({key => value, 'name' => test_name}, zlevel, on_water, 'node')
+        with_name = Scene.new({ key => value, 'name' => test_name }, zlevel, on_water, 'node', true)
         with_name.flush_cache
         puts 'calculating'
       end
-      #puts key+"="+value+" - name is OK for name '#{name}'"
+      # puts key+"="+value+" - name is OK for name '#{name}'"
     end
 
-    def is_object_displaying_name(key, value, name, zlevel, on_water)
-      name_tags = {key => value, 'name' => name}
-      with_name = Scene.new(name_tags, zlevel, on_water, 'node')
-      nameless_tags = {key => value}
-      nameless = Scene.new(nameless_tags, zlevel, on_water, 'node')
+    def is_object_displaying_name_as_this_object_type(tags, name, zlevel, on_water, type)
+      name_tags = tags.merge({ 'name' => name })
+      with_name = Scene.new(name_tags, zlevel, on_water, type, true)
+      nameless_tags = tags
+      nameless = Scene.new(nameless_tags, zlevel, on_water, type, true)
       return with_name.is_output_different(nameless)
     end
 
-    def is_object_displaying_anything(key, value, zlevel, on_water)
-      object = Scene.new({key => value}, zlevel, on_water, 'node')
-      empty = Scene.new({}, zlevel, on_water, 'node')
+    def is_object_displaying_name_as_node(tags, name, zlevel, on_water)
+      is_object_displaying_name_as_this_object_type(tags, name, zlevel, on_water, 'node')
+    end
+
+    def is_object_displaying_anything_as_this_object_type(tags, zlevel, on_water, type)
+      object = Scene.new(tags, zlevel, on_water, type, true)
+      empty = Scene.new({}, zlevel, on_water, type, true)
       return object.is_output_different(empty)
     end
+
+    def is_object_displaying_anything_as_node(tags, zlevel, on_water)
+      return is_object_displaying_anything_as_this_object_type(tags, zlevel, on_water, 'node')
+    end
   end
 end

data/lib/cartocss_helper/visualise_changes_diff_from_images.rb

@@ -7,9 +7,10 @@ module CartoCSSHelper
       @file_location = file_location
       @description = description
     end
+
     def identical(another_image)
-      #Returns true if the contents of a file A and a file B are identical.
-      return FileUtils.compare_file(self.file_location, another_image.file_location)
+      # Returns true if the contents of a file A and a file B are identical.
+      return FileUtils.compare_file(file_location, another_image.file_location)
     end
   end
 
@@ -22,6 +23,7 @@ module CartoCSSHelper
       @right_file_location = right_image.file_location
       @description = left_image.description
     end
+
     def merge_description_from_next_image(image)
       @description << ', ' << image.description
     end
@@ -36,7 +38,7 @@ module CartoCSSHelper
       @image_size = image_size
       @margin = 10
       @standard_pointsize = 10
-      @header_space = @standard_pointsize*1.5
+      @header_space = @standard_pointsize * 1.5
       @diff_note_space = @standard_pointsize
 
       render
@@ -44,13 +46,13 @@ module CartoCSSHelper
 
     def compress(before, after)
       returned = [PairOfComparedImages.new(before[0], after[0])]
-      (1...before.length).each { |i|
-        if before[i].identical(before[i-1]) && after[i].identical(after[i-1])
+      (1...before.length).each do |i|
+        if before[i].identical(before[i - 1]) && after[i].identical(after[i - 1])
           returned[-1].merge_description_from_next_image(before[i])
         else
           returned.push(PairOfComparedImages.new(before[i], after[i]))
         end
-      }
+      end
       return returned
     end
 
@@ -58,17 +60,16 @@ module CartoCSSHelper
       create_canvas
       offset = 0
       render_header
-      offset += @header_space + @margin*1.5
-      render_diff_note offset
-      offset += @diff_note_space + @margin*0.5
+      offset += @header_space + @margin * 1.5
+      offset += @diff_note_space + @margin * 0.5
       render_images_with_labels offset
-      offset += (@compared.length)*(@margin + @image_size)
+      offset += @compared.length * (@margin + @image_size)
       render_footer offset
     end
 
     def create_canvas
       y = get_needed_y
-      x = @image_size*2+3*@margin
+      x = @image_size * 2 + 3 * @margin
 
       @canvas = Magick::Image.new(x, y)
     end
@@ -76,48 +77,49 @@ module CartoCSSHelper
     def get_needed_y
       y = 0
       y += @header_space
-      y += @margin*1.5
+      y += @margin * 1.5
       y += @diff_note_space
-      y += @margin*0.5
-      y += (@compared.length) * (@image_size + @margin)
+      y += @margin * 0.5
+      y += @compared.length * (@image_size + @margin)
       y += @diff_note_space
       return y
     end
 
     def render_header
       header_drawer = Magick::Draw.new
-      header_drawer.pointsize(@header_space*3/5)
+      header_drawer.pointsize(@header_space * 3 / 5)
       header_drawer.text(@margin, @header_space, @header)
       header_drawer.draw(@canvas)
     end
 
-    def render_diff_note(y_offset)
-      drawer = Magick::Draw.new
-      drawer.pointsize(@diff_note_space)
-      #drawer.text(@margin, y_offset, 'before')
-      drawer.draw(@canvas)
-      #drawer.text(@margin*2 + @image_size, y_offset, 'after')
-      drawer.draw(@canvas)
-    end
-
     def render_images_with_labels(y_offset)
-      (0...@compared.length).each { |i|
+      (0...@compared.length).each do |i|
         render_row_of_labelled_images(@compared[i], y_offset + i * (@margin + @image_size))
-      }
+      end
+    end
+
+    def read_image_from_disk(filename)
+      return Magick::Image.read(filename)[0]
+    rescue Magick::ImageMagickError => e
+      # for example on loading corrupted empty file (for example CartoCSSHelper was interuppted during writing to disk...)
+      puts "reading image <#{filename}> from disk failed"
+      puts e.class
+      puts e
+      raise e
     end
 
     def render_row_of_labelled_images(processed, y_offset)
-      left_image = Magick::Image.read(processed.left_file_location)[0]
-      right_image = Magick::Image.read(processed.right_file_location)[0]
+      left_image = read_image_from_disk(processed.left_file_location)
+      right_image = read_image_from_disk(processed.right_file_location)
       drawer = Magick::Draw.new
-      drawer.pointsize(@diff_note_space*4/5)
+      drawer.pointsize(@diff_note_space * 4 / 5)
       if left_image == right_image
-        drawer.text(@margin + @image_size/2, y_offset, "#{processed.description} - unchanged")
+        drawer.text(@margin + @image_size / 2, y_offset, "#{processed.description} - unchanged")
         drawer.draw(@canvas)
       else
         drawer.text(@margin, y_offset, "#{processed.description} - before")
         drawer.draw(@canvas)
-        drawer.text(@margin*2 + @image_size, y_offset, "#{processed.description} - after")
+        drawer.text(@margin * 2 + @image_size, y_offset, "#{processed.description} - after")
         drawer.draw(@canvas)
       end
       render_row_of_images(y_offset, left_image, right_image)
@@ -133,17 +135,17 @@ module CartoCSSHelper
     # noinspection RubyResolve
     def render_row_of_images(y_offset, left_image, right_image)
       if left_image == right_image
-        @canvas.composite!(left_image, @margin*1.5 + @image_size/2, y_offset, Magick::OverCompositeOp)
+        @canvas.composite!(left_image, @margin * 1.5 + @image_size / 2, y_offset, Magick::OverCompositeOp)
       else
         @canvas.composite!(left_image, @margin, y_offset, Magick::OverCompositeOp)
-        @canvas.composite!(right_image, @margin*2+@image_size, y_offset, Magick::OverCompositeOp)
+        @canvas.composite!(right_image, @margin * 2 + @image_size, y_offset, Magick::OverCompositeOp)
       end
     end
 
     def render_footer(y_offset)
       label_drawer = Magick::Draw.new
       label_drawer.pointsize(@standard_pointsize)
-      label_drawer.text(@margin, y_offset, 'generated using https://github.com/mkoniecz/CartoCSSHelper')
+      label_drawer.text(@margin, y_offset, 'generated using https://github.com/matkoniecz/CartoCSSHelper')
       label_drawer.draw(@canvas)
     end