cartocss_helper 1.0.1 → 1.2.1

data/lib/cartocss_helper/data_file_handling.rb

@@ -10,11 +10,13 @@ module CartoCSSHelper
       end
       return @@loaded_filename
     end
+
     def self.load_data_into_database(data_filename, debug=false)
       if get_filename_of_recently_loaded_file == data_filename
         puts "\tavoided reloading the same file! <#{data_filename}>"
         return
       end
+      start_time = Time.now
       puts "\tloading data into database <#{data_filename}>"
       @@loaded_filename = nil
       silence = '> /dev/null 2>&1'
@@ -36,6 +38,8 @@ module CartoCSSHelper
         end
       end
       @@loaded_filename = data_filename
+      time_in_seconds = Time.now - start_time
+      puts "\tloading lasted #{time_in_seconds.to_i}s"
     end
   end
 

data/lib/cartocss_helper/downloader.rb

@@ -11,14 +11,13 @@ module CartoCSSHelper
       return get_overpass_query_results_file_location(query, "download data for #{latitude} #{longitude} (#{size})")
     end
 
-
     def self.download_osm_data_for_location(latitude, longitude, size, accept_cache=true)
       filename = CartoCSSHelper::Configuration.get_path_to_folder_for_cache + "#{latitude} #{longitude} #{size}.osm"
       if File.exists?(filename)
         if accept_cache
           return filename
         end
-        File.delete(filename)
+        delete_file(filename, 'query refusing to accept cache was used')
       end
       query = get_query_to_download_data_around_location(latitude, longitude, size)
       text = get_overpass_query_results(query, "download data for #{latitude} #{longitude} (#{size})")
@@ -38,7 +37,7 @@ module CartoCSSHelper
       query += "\n"
       query += "(node(#{bb});<;);"
       query += "\n"
-      query += 'out meta;'
+      query += 'out;'
       query += "\n"
       query += '/*'
       query += "\nbbox size: #{size}"
@@ -134,7 +133,9 @@ module CartoCSSHelper
     def self.get_overpass_query_results(query, description, debug=false)
       cached = get_overpass_query_results_from_cache(query)
       if cached == ''
-        raise OverpassRefusedResponse
+        if File.exists?(get_query_cache_refused_response_filename(query))
+          raise OverpassRefusedResponse
+        end
       end
       return cached unless cached == nil
 
@@ -148,6 +149,7 @@ module CartoCSSHelper
       begin
         cached = Downloader.run_overpass_query query, description
       rescue OverpassRefusedResponse
+        mark_query_as_refused(query)
         write_to_cache(query, '')
         raise OverpassRefusedResponse
       end
@@ -155,6 +157,16 @@ module CartoCSSHelper
       return cached
     end
 
+    def self.get_query_cache_refused_response_filename(query)
+      return get_query_cache_filename(query)+'_response_refused'
+    end
+
+    def self.mark_query_as_refused(query)
+      file = File.new(get_query_cache_refused_response_filename(query), 'w')
+      file.write ''
+      file.close
+    end
+
     def self.write_to_cache(query, response)
       file = File.new(get_query_cache_filename(query), 'w')
       file.write response
@@ -199,7 +211,7 @@ module CartoCSSHelper
     end
 
     def self.not_enough_free_space
-      minimum_gb = 2
+      minimum_gb = 1
       return get_available_space_for_cache_in_gb < minimum_gb
     end
 
@@ -210,22 +222,30 @@ module CartoCSSHelper
 
     def self.attempt_cleanup
       if not_enough_free_space
-        delete_large_overpass_caches
+        delete_large_overpass_caches 500
+      end
+      if not_enough_free_space
+        delete_large_overpass_caches 100
+      end
+      if not_enough_free_space
+        delete_large_overpass_caches 50
       end
     end
 
-    def self.delete_file(file)
-      open(CartoCSSHelper::Configuration.get_path_to_folder_for_cache+'log.txt', 'a') { |file|
-        file.puts("deleting #{file}, #{File.size(file)/1024/1024}MB")
+    def self.delete_file(filename, reason)
+      open(CartoCSSHelper::Configuration.get_path_to_folder_for_output+'log.txt', 'a') { |file|
+        message = "deleting #{filename}, #{File.size(filename)/1024/1024}MB - #{reason}"
+        puts message
+        file.puts(message)
+        File.delete(filename)
       }
-      File.delete(file)
     end
 
-    def self.delete_large_overpass_caches
+    def self.delete_large_overpass_caches(threshold_in_MB)
       #todo - find library that deals with caches like this, bug here may be unfunny
       Dir.glob(CartoCSSHelper::Configuration.get_path_to_folder_for_overpass_cache+'*') {|file|
-        if File.size(file) > (1024 * 1024 * 50)
-          delete_file(file)
+        if File.size(file) > (1024 * 1024 * threshold_in_MB)
+          delete_file(file, "removing everpass cache entries larger than #{threshold_in_MB} MB to make free space on the disk")
         end
       }
     end

data/lib/cartocss_helper/heuristic.rb

@@ -98,7 +98,7 @@ module CartoCSSHelper
       return tags
     end
 
-    def get_generic_tag_value
+    def self.get_generic_tag_value
       return '*'
     end
 

data/lib/cartocss_helper/image_generator.rb

@@ -87,7 +87,7 @@ module CartoCSSHelper
     def generate_image(lat, lon, debug)
       export_filename = self.get_filename
       bbox_size = self.get_bbox_size
-      TilemillHandler.run_tilemill_export_image(lat, lon,  @zlevel, bbox_size, 200, export_filename, debug)
+      TilemillHandler.run_tilemill_export_image(lat, lon,  @zlevel, [bbox_size, bbox_size], 200, export_filename, debug)
     end
   end
 end

data/lib/cartocss_helper/style_specific/default_osm_style.rb

@@ -154,7 +154,6 @@ module CartoCSSHelper
             TagRenderingStatus.new('highway', 'platform', :primary),
             TagRenderingStatus.new('highway', 'primary', :primary),
             TagRenderingStatus.new('highway', 'primary_link', :primary),
-            TagRenderingStatus.new('highway', 'proposed', :primary),
             TagRenderingStatus.new('highway', 'raceway', :primary),
             TagRenderingStatus.new('highway', 'residential', :primary),
             TagRenderingStatus.new('highway', 'rest_area', :primary),

data/lib/cartocss_helper/tag_lister.rb

@@ -104,7 +104,7 @@ module CartoCSSHelper
       [false, true].each { |on_water|
         [Configuration.get_max_z].each { |zlevel|
           ['area', 'closed_way', 'way', 'node'].each{ |type|
-            if rendered_on_zlevel({key => value}, type, zlevel, on_water)
+            if CartoCSSHelper::Info.rendered_on_zlevel({key => value}, type, zlevel, on_water)
               if !is_key_rendered_and_value_ignored_set(key, value, type, zlevel, on_water)
                 return false
               end
@@ -123,9 +123,9 @@ module CartoCSSHelper
 
     def is_rendered(key, value)
       [false, true].each { |on_water|
-        [Configuration.get_max_z].each { |zlevel|
+        [Configuration.get_max_z].each { |zlevel| #TODO - note that some tags may be rendered up to X zoom level, but checking all zlevels would take too much time
           ['area', 'closed_way', 'way', 'node'].each{ |type|
-            if rendered_on_zlevel({key => value}, type, zlevel, on_water)
+            if CartoCSSHelper::Info.rendered_on_zlevel({key => value}, type, zlevel, on_water)
               return true
             end
           }
@@ -142,8 +142,6 @@ module CartoCSSHelper
       return true
     end
 
-    protected
-
     def how_rendered_as_composite(key, value, suggested_composite)
       [false, true].each { |on_water|
         [Configuration.get_max_z].each { |zlevel|
@@ -167,12 +165,14 @@ module CartoCSSHelper
       return nil
     end
 
-    def rendered_on_zlevel(tags, type, zlevel, on_water)
+    def self.rendered_on_zlevel(tags, type, zlevel, on_water)
       empty = Scene.new({}, zlevel, on_water, type)
       tested = Scene.new(tags, zlevel, on_water, type)
       return tested.is_output_different(empty)
     end
 
+    protected
+
     def how_rendered_on_zlevel_as_composite(tags, type, zlevel, on_water, suggested_composite)
       if suggested_composite != nil
         if is_rendered_with_this_composite tags, type, suggested_composite, zlevel, on_water

data/lib/cartocss_helper/tilemill_handler.rb

@@ -11,11 +11,13 @@ module CartoCSSHelper
       if debug
         silence = ''
       end
+      latitude_bb_size = bbox_size[0]
+      longitude_bb_size = bbox_size[0]
       #--bbox=[xmin,ymin,xmax,ymax]
-      xmin = lon-bbox_size/2
-      ymin = lat-bbox_size/2
-      xmax = lon+bbox_size/2
-      ymax = lat+bbox_size/2
+      xmin = lon-longitude_bb_size/2
+      ymin = lat-latitude_bb_size/2
+      xmax = lon+longitude_bb_size/2
+      ymax = lat+latitude_bb_size/2
       bbox = "#{xmin},#{ymin},#{xmax},#{ymax}"
       params = "--format=png --width=#{image_size} --height=#{image_size} --static_zoom=#{zlevel} --bbox=\"#{bbox}\""
       project_name = CartoCSSHelper::Configuration.get_tilemill_project_name

data/lib/cartocss_helper/visualise_changes_diff_from_images.rb

@@ -1,4 +1,4 @@
-require 'RMagick'
+require 'rmagick'
 
 module CartoCSSHelper
   class ImageForComparison

data/lib/cartocss_helper/visualise_changes_image_generation.rb

@@ -47,8 +47,8 @@ module CartoCSSHelper
       end
     end
 
-    def self.add_job(filename, latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size)
-      print 'pool <- '
+    def self.add_job(filename, latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size, prefix)
+      print prefix
       new_job = MapGenerationJob.new(filename, latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size)
       new_job.print
 
@@ -79,8 +79,8 @@ module CartoCSSHelper
       @@jobs = []
     end
 
-    def self.shuffle_jobs
-      @@jobs.shuffle!
+    def self.shuffle_jobs(seed)
+      @@jobs.shuffle!(random: Random.new(seed))
     end
 
     def self.visualise_changes_synthethic_test(tags, type, on_water, zlevel_range, new_branch, old_branch)
@@ -143,17 +143,19 @@ module CartoCSSHelper
 
     def self.visualise_changes_on_real_data(tags, type, wanted_latitude, wanted_longitude, zlevels, new_branch, old_branch='master')
       #special support for following tag values:  :any_value
-      header = "#{ VisualDiff.dict_to_pretty_tag_list(tags) } #{type} #{ wanted_latitude } #{ wanted_longitude } #{zlevels}"
-      puts "visualise_changes_on_real_data <#{header}> #{old_branch} -> #{new_branch}"
+      header_prefix = "#{ VisualDiff.dict_to_pretty_tag_list(tags) } #{type} [#{ wanted_latitude }, #{ wanted_longitude }] -> "
+      target_location = '[?, ?]'
+      header_sufix = " #{old_branch}->#{new_branch} #{zlevels}"
+      puts "visualise_changes_on_real_data <#{header_prefix}#{header_sufix}> #{old_branch} -> #{new_branch}"
       begin
         latitude, longitude = Downloader.locate_element_with_given_tags_and_type tags, type, wanted_latitude, wanted_longitude
+        target_location = "[#{latitude}, #{longitude}]"
       rescue Downloader::OverpassRefusedResponse
         puts 'No nearby instances of tags and tag is not extremely rare - no generation of nearby location and wordwide search was impossible. No diff image will be generated for this location.'
         return false
       end
-      header = "#{ VisualDiff.dict_to_pretty_tag_list(tags) } #{type} #{ wanted_latitude } #{ wanted_longitude } #{zlevels}"
       download_bbox_size = 0.4
-      visualise_changes_for_location(latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size)
+      visualise_changes_for_location(latitude, longitude, zlevels, header_prefix+target_location+header_sufix, new_branch, old_branch, download_bbox_size)
       return true
     end
 
@@ -167,7 +169,7 @@ module CartoCSSHelper
       if @@job_pooling
         prefix = 'pool <- '
       end
-      add_job(filename, latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size)
+      add_job(filename, latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size, prefix)
       if !@@job_pooling
         run_jobs
       end
@@ -187,27 +189,17 @@ module CartoCSSHelper
       return reference_value*rescaler
     end
 
-    def self.collect_images_for_real_data_test(latitude, longitude, zlevels, source, wanted_image_size=400)
+    def self.get_render_bbox_size(zlevel, wanted_image_size, latitude)
+      longitude_equator_rendered_length_in_pixels = 256 * 2**zlevel
+      longitude_size = 360*wanted_image_size.to_f/longitude_equator_rendered_length_in_pixels
+      latitude_size = longitude_size * Math.cos(latitude*Math::PI/180)
+      return [latitude_size, longitude_size]
+    end
+
+    def self.collect_images_for_real_data_test(latitude, longitude, zlevels, source, image_size=400)
       collection = []
       zlevels.each { |zlevel|
-        image_size_for_16_zoom_level = 1000
-        image_size = (VisualDiff.scale zlevel, image_size_for_16_zoom_level, 16)
-        mutiplier = 1000
-        image_size = (image_size*mutiplier).to_int
-        render_bbox_size = 0.015
-        ratio = 1.0*image_size/(wanted_image_size*mutiplier)
-        image_size /= ratio
-        render_bbox_size /= ratio
-        image_size /= mutiplier
-        image_size = image_size.to_i
-        if image_size!=wanted_image_size
-          puts VisualDiff.scale zlevel, image_size_for_16_zoom_level, 16
-          puts zlevel
-          puts image_size
-          puts wanted_image_size
-          puts ratio
-          raise "#{image_size} mismatches #{wanted_image_size}"
-        end
+        render_bbox_size = VisualDiff.get_render_bbox_size(zlevel, image_size, latitude)
         cache_folder = CartoCSSHelper::Configuration.get_path_to_folder_for_branch_specific_cache
         filename = "#{cache_folder+"#{latitude} #{longitude} #{zlevel}zlevel #{image_size}px #{source.get_timestamp} #{source.download_bbox_size}.png"}"
         if !File.exists?(filename)

data/lib/cartocss_helper.rb

@@ -14,33 +14,41 @@ include CartoCSSHelper::Validator
 include CartoCSSHelper::Git
 
 module CartoCSSHelper
-  def  self.test_tag_on_real_data(tags, new_branch, old_branch, zlevels, types=['node', 'closed_way', 'way'])
+  def self.test_tag_on_real_data(tags, new_branch, old_branch, zlevels, types=['node', 'closed_way', 'way'], min = 4, skip = 0)
     types.each {|type|
-      test_tag_on_real_data_for_this_type(tags, new_branch, old_branch, zlevels, type)
+      test_tag_on_real_data_for_this_type(tags, new_branch, old_branch, zlevels, type, min, skip)
     }
   end
 
-  def self.test_tag_on_real_data_for_this_type(tags, new_branch, old_branch, zlevels, type)
-    min = 6
+  def self.test_tag_on_real_data_for_this_type(tags, new_branch, old_branch, zlevels, type, min = 4, skip = 0)
+    if type.kind_of?(Array)
+      type = type[0]
+    end
     generated = 0
 
     n = 0
     max_n = get_maxn_for_nth_location
+    max_n -= skip
+    skip_string = ''
+    if skip > 0
+      skip_string = " (#{skip} locations skipped)"
+    end
     while generated < min
-      location = get_nth_location(n)
+      location = get_nth_location(n + skip)
       generated +=1 if CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, location[0], location[1], zlevels, new_branch, old_branch)
       n+=1
       if n > max_n
         return
       end
+      puts "#{n}/#{max_n} locations checked #{skip_string}. #{generated}/#{min} testing location found"
     end
   end
 
-  def get_maxn_for_nth_location
+  def self.get_maxn_for_nth_location
     return get_list_of_testing_locations.length-1
   end
 
-  def get_nth_location(n)
+  def self.get_nth_location(n)
     return get_list_of_testing_locations[n]
   end
 
@@ -70,29 +78,40 @@ module CartoCSSHelper
     }
   end
 
-  def visualise_place_by_url(url, zlevels, new_branch, old_branch='master', header=nil, download_bbox_size=0.04, image_size = 350)
-    raise "#{url} is not a string" unless old_branch.class == String
-    raise "#{zlevels} is not a range" unless zlevels.class == Range
-    raise "#{new_branch} is not a string" unless new_branch.class == String
-    raise "#{old_branch} is not a string" unless old_branch.class == String
-    raise "#{header} is not a string" unless header.class == String
-    raise "#{download_bbox_size} is not a number" unless download_bbox_size.kind_of? Numeric
-    raise "#{image_size} is not a integer" unless image_size.kind_of? Integer
+  def self.get_latitude_longitude_from_url(url)
+    if(url.scan(/[\/]((-|)\d+(\.\d+))/)).length >= 2
+      latitude = url.scan(/[\/]((-|)\d+(\.\d+))/)[0][0].to_f
+      longitude = url.scan(/[\/]((-|)\d+(\.\d+))/)[1][0].to_f
+      return latitude, longitude
+    end
+    latitude = url.scan(/[\/=]((-|)\d+(\.\d+))/)[0][0].to_f
+    longitude = url.scan(/[\/=]((-|)\d+(\.\d+))/)[1][0].to_f
+    return latitude, longitude
+  end
 
-    latitude = url.scan(/\/((-|)\d+(\.\d+))/)[0][0].to_f
-    longitude = url.scan(/\/((-|)\d+(\.\d+))/)[1][0].to_f
+  def self.visualise_place_by_url(url, zlevels, new_branch, old_branch='master', header=nil, download_bbox_size=0.04, image_size = 350)
     if header == nil
       header = url
     end
+
+    raise "#{url} is not a string, it is #{url.class}" unless url.class == String
+    raise "#{zlevels} is not a range, it is #{zlevels.class}" unless zlevels.class == Range
+    raise "#{new_branch} is not a string, it is #{new_branch.class}" unless new_branch.class == String
+    raise "#{old_branch} is not a string, it is #{old_branch.class}" unless old_branch.class == String
+    raise "#{header} is not a string, it is #{header.class}" unless header.class == String
+    raise "#{download_bbox_size} is not a number" unless download_bbox_size.kind_of? Numeric
+    raise "#{image_size} is not a integer" unless image_size.kind_of? Integer
+
+    latitude, longitude = get_latitude_longitude_from_url(url)
     header += ' ' + old_branch + '->' + new_branch + ' ' + zlevels.to_s + ' '+ image_size.to_s + 'px'
     CartoCSSHelper::VisualDiff.visualise_changes_for_location(latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size)
   end
 
-  def get_place_of_storage_of_resource_under_url(url)
+  def self.get_place_of_storage_of_resource_under_url(url)
     return CartoCSSHelper::Configuration.get_path_to_folder_for_overpass_cache + '.manual.cache' + FileHelper::make_string_usable_as_filename(url)
   end
 
-  def download_remote_file(url)
+  def self.download_remote_file(url)
     filename = get_place_of_storage_of_resource_under_url(url)
     if !File.exists?(filename)
       begin
@@ -109,13 +128,13 @@ module CartoCSSHelper
     end
   end
 
-  def visualise_place_by_remote_file(url, latitude, longitude, zlevels, new_branch, old_branch='master', header=nil, bb=0.04, image_size = 350)
+  def self.visualise_place_by_remote_file(url, latitude, longitude, zlevels, new_branch, old_branch='master', header=nil, bb=0.04, image_size = 350)
     download_remote_file(url)
     filename = get_place_of_storage_of_resource_under_url(url)
     visualise_place_by_file(filename, latitude, longitude, zlevels, new_branch, old_branch, header, bb, image_size)
   end
 
-  def visualise_place_by_file(filename, latitude, longitude, zlevels, new_branch, old_branch='master', header=nil, bb=0.04, image_size = 350)
+  def self.visualise_place_by_file(filename, latitude, longitude, zlevels, new_branch, old_branch='master', header=nil, bb=0.04, image_size = 350)
     raise "#{filename} does not exists" unless File.exists?(filename)
     raise "#{latitude} is not a number" unless latitude.kind_of? Numeric
     raise "#{longitude} is not a number" unless longitude.kind_of? Numeric

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cartocss_helper
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-16 00:00:00.000000000 Z
+date: 2015-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.7.3
+        version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.7.3
+        version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: sys-filesystem
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.13.4
+        version: 2.15.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.13.4
+        version: 2.15.2
 description: Tool to make development of CartoCSS styles more efficient. Automates
   actions necessary to produce test images and validates style. Loading data using
   osm2pgsql, rendering with TileMill, obtaining test data from overpass turbo.