cartocss_helper 0.0.0 → 1.0.0

data/lib/cartocss_helper.rb

@@ -9,62 +9,127 @@ require_relative 'cartocss_helper/downloader.rb'
 require_relative 'cartocss_helper/data_file_handling.rb'
 require_relative 'cartocss_helper/validator.rb'
 require_relative 'cartocss_helper/git.rb'
+require_relative 'data/testing_locations'
 include CartoCSSHelper::Validator
 include CartoCSSHelper::Git
 
 module CartoCSSHelper
-  def  self.test_tag_on_real_data(tags, zlevels, old_branch, new_branch)
-    ['node', 'closed_way', 'way'].each {|type|
-      test_tag_on_real_data_for_this_type(tags, zlevels, old_branch, new_branch, type)
+  def  self.test_tag_on_real_data(tags, new_branch, old_branch, zlevels, types=['node', 'closed_way', 'way'])
+    types.each {|type|
+      test_tag_on_real_data_for_this_type(tags, new_branch, old_branch, zlevels, type)
     }
   end
 
-  def self.test_tag_on_real_data_for_this_type(tags, zlevels, old_branch, new_branch, type)
-    #special support for following tag values:  :any_value
-    krakow_latitude = 50.1
-    krakow_longitude = 19.9
-    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, krakow_latitude, krakow_longitude, zlevels, old_branch, new_branch)
-    japan_latitude = 36.1
-    japan_longitude = 140.7
-    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, japan_latitude, japan_longitude, zlevels, old_branch, new_branch)
-    russia_latitude = 54.8
-    russia_longitude = 31.7
-    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, russia_latitude, russia_longitude, zlevels, old_branch, new_branch)
-    mecca_latitude = 21.3
-    mecca_longitude = 39.5
-    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, mecca_latitude, mecca_longitude, zlevels, old_branch, new_branch)
-    georgia_latitude = 41.4
-    georgia_longitude = 44.5
-    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, georgia_latitude, georgia_longitude, zlevels, old_branch, new_branch)
-    london_latitude = 51.5
-    london_longitude = -0.1
-    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, london_latitude, london_longitude, zlevels, old_branch, new_branch)
-    #TODO: solve problems with respons to big to store in memory
-    #utrecht_latitude =  52.09
-    #utrecht_longitude = 5.11
-    #CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, utrecht_latitude, utrecht_longitude, zlevels, old_branch, new_branch)
-    rural_uk_latitude =  53.2
-    rural_uk_longitude = -1.8
-    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, rural_uk_latitude, rural_uk_longitude, zlevels, old_branch, new_branch)
+  def self.test_tag_on_real_data_for_this_type(tags, new_branch, old_branch, zlevels, type)
+    min = 6
+    generated = 0
+
+    n = 0
+    max_n = get_maxn_for_nth_location
+    while generated < min
+      location = get_nth_location(n)
+      generated +=1 if CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, location[0], location[1], zlevels, new_branch, old_branch)
+      n+=1
+      if n > max_n
+        return
+      end
+    end
+  end
+
+  def get_maxn_for_nth_location
+    return get_list_of_testing_locations.length-1
   end
 
+  def get_nth_location(n)
+    return get_list_of_testing_locations[n]
+  end
+
+
   def self.add_common_secondary_tags(tags)
-    return tags.merge({'name' => 'ÉÉÉÉÉÉ ÉÉÉÉÉÉ ÉÉÉÉÉÉ', 'ref' => '1', 'ele' => '8000', 'operator' => 'operator ÉÉ ÉÉ ÉÉ', 'brand' => 'brand ÉÉ ÉÉ ÉÉ ÉÉ'})
+    added_tags = {'name' => 'ÉÉÉÉÉÉ ÉÉÉÉÉÉ', 'ref' => '1', 'ele' => '8000', 'operator' => 'ÉÉ ÉÉ ÉÉ operator', 'brand' => 'ÉÉ ÉÉ ÉÉ brand'}
+    return tags.merge(added_tags)
   end
 
-  def self.test(tags, new_branch, old_brach='master', test_on_water=false, zlevels=Configuration.get_min_z..Configuration.get_max_z)
-    puts "processing #{VisualDiff.dict_to_pretty_tag_list(tags)}"
+  def self.test_tag_on_sythetic_data(tags, new_branch, old_branch='master', zlevels=Configuration.get_min_z..Configuration.get_max_z, types=['node', 'closed_way', 'way'], test_on_water=false)
     syn_tags = add_common_secondary_tags(tags)
-    ['node', 'closed_way', 'way'].each {|type|
-      CartoCSSHelper::VisualDiff.visualise_changes_synthethic_test(syn_tags, type, test_on_water, zlevels, old_brach, new_branch)
+    types.each {|type|
+      CartoCSSHelper::VisualDiff.visualise_changes_synthethic_test(syn_tags, type, test_on_water, zlevels, new_branch, old_branch)
     }
-    test_tag_on_real_data(tags, zlevels, old_brach, new_branch)
   end
 
-  def self.probe(tags, new_branch, old_brach='master',test_on_water=false, zlevels=Configuration.get_min_z..Configuration.get_max_z)
+  def self.test(tags, new_branch, old_branch='master', zlevels=Configuration.get_min_z..Configuration.get_max_z, types=['node', 'closed_way', 'way'], test_on_water=false)
+    puts "processing #{VisualDiff.dict_to_pretty_tag_list(tags)}"
+    test_tag_on_sythetic_data(tags, new_branch, old_branch, zlevels, types, test_on_water)
+    test_tag_on_real_data(tags, new_branch, old_branch, zlevels, types)
+  end
+
+  def self.probe(tags, new_branch, old_branch='master', zlevels=Configuration.get_min_z..Configuration.get_max_z, types=['node', 'closed_way', 'way'], test_on_water=false)
     syn_tags = add_common_secondary_tags(tags)
-    ['node', 'closed_way', 'way'].each {|type|
-      CartoCSSHelper::VisualDiff.visualise_changes_synthethic_test(syn_tags, type, test_on_water, zlevels, old_brach, new_branch)
+    types.each {|type|
+      CartoCSSHelper::VisualDiff.visualise_changes_synthethic_test(syn_tags, type, test_on_water, zlevels, new_branch, old_branch)
     }
   end
-end
+
+  def visualise_place_by_url(url, zlevels, new_branch, old_branch='master', header=nil, download_bbox_size=0.04, image_size = 350)
+    raise "#{url} is not a string" unless old_branch.class == String
+    raise "#{zlevels} is not a range" unless zlevels.class == Range
+    raise "#{new_branch} is not a string" unless new_branch.class == String
+    raise "#{old_branch} is not a string" unless old_branch.class == String
+    raise "#{header} is not a string" unless header.class == String
+    raise "#{download_bbox_size} is not a number" unless download_bbox_size.kind_of? Numeric
+    raise "#{image_size} is not a integer" unless image_size.kind_of? Integer
+
+    latitude = url.scan(/\/((-|)\d+(\.\d+))/)[0][0].to_f
+    longitude = url.scan(/\/((-|)\d+(\.\d+))/)[1][0].to_f
+    if header == nil
+      header = url
+    end
+    header += ' ' + old_branch + '->' + new_branch + ' ' + zlevels.to_s + ' '+ image_size.to_s + 'px'
+    CartoCSSHelper::VisualDiff.visualise_changes_for_location(latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size)
+  end
+
+  def get_place_of_storage_of_resource_under_url(url)
+    return CartoCSSHelper::Configuration.get_path_to_folder_for_overpass_cache + '.manual.cache' + FileHelper::make_string_usable_as_filename(url)
+  end
+
+  def download_remote_file(url)
+    filename = get_place_of_storage_of_resource_under_url(url)
+    if !File.exists?(filename)
+      begin
+        url = url
+        timeout = 600
+        data = RestClient::Request.execute(:method => :get, :url => url, :timeout => timeout)
+      rescue => e
+        puts "visualise_place_by_remote_file failed to fetch #{url}"
+        raise e
+      end
+      file = File.new(filename, 'w')
+      file.write data
+      file.close
+    end
+  end
+
+  def visualise_place_by_remote_file(url, latitude, longitude, zlevels, new_branch, old_branch='master', header=nil, bb=0.04, image_size = 350)
+    download_remote_file(url)
+    filename = get_place_of_storage_of_resource_under_url(url)
+    visualise_place_by_file(filename, latitude, longitude, zlevels, new_branch, old_branch, header, bb, image_size)
+  end
+
+  def visualise_place_by_file(filename, latitude, longitude, zlevels, new_branch, old_branch='master', header=nil, bb=0.04, image_size = 350)
+    raise "#{filename} does not exists" unless File.exists?(filename)
+    raise "#{latitude} is not a number" unless latitude.kind_of? Numeric
+    raise "#{longitude} is not a number" unless longitude.kind_of? Numeric
+    raise "#{zlevels} is not a range" unless zlevels.class == Range
+    raise "#{new_branch} is not a string" unless new_branch.class == String
+    raise "#{old_branch} is not a string" unless old_branch.class == String
+    raise "#{header} is not a string" unless header.class == String
+    raise "#{bb} is not a number" unless bb.kind_of? Numeric
+    raise "#{image_size} is not a integer" unless image_size.kind_of? Integer
+
+    if header == nil
+      header = filename
+    end
+    header += ' ' + old_branch + '->' + new_branch + '[' + latitude.to_s + ',' + longitude.to_s + ']' + ' ' + image_size.to_s + 'px'
+    CartoCSSHelper::VisualDiff.visualise_changes_for_location_from_file(filename, latitude, longitude, zlevels, header, new_branch, old_branch, bb, image_size)
+  end
+end

data/lib/cartocss_helper/configuration.rb

@@ -86,7 +86,7 @@ module CartoCSSHelper::Configuration
   end
 
 	def get_path_to_folder_for_branch_specific_cache
-		location = File.join(get_path_to_folder_for_cache, CartoCSSHelper::Git.get_commit_hash, '')
+		location = File.join(get_path_to_folder_for_cache, 'generated_images', CartoCSSHelper::Git.get_commit_hash, '')
 		FileUtils::mkdir_p location
 		return location
 	end

data/lib/cartocss_helper/data_file_handling.rb

@@ -3,7 +3,20 @@ require_relative 'configuration'
 
 module CartoCSSHelper
   class DataFileLoader
+    @@loaded_filename = nil
+    def self.get_filename_of_recently_loaded_file
+      if @@loaded_filename == Configuration.get_data_filename
+        return nil
+      end
+      return @@loaded_filename
+    end
     def self.load_data_into_database(data_filename, debug=false)
+      if get_filename_of_recently_loaded_file == data_filename
+        puts "\tavoided reloading the same file! <#{data_filename}>"
+        return
+      end
+      puts "\tloading data into database <#{data_filename}>"
+      @@loaded_filename = nil
       silence = '> /dev/null 2>&1'
       if debug
         silence = ''
@@ -13,7 +26,16 @@ module CartoCSSHelper
       if debug
         puts command
       end
-      system command
+      if !system(command)
+        puts 'loading data into database failed'
+        if !debug
+          puts 'retry with enabled debug'
+          load_data_into_database(data_filename, true)
+        else
+          raise 'osm2pgsql failed'
+        end
+      end
+      @@loaded_filename = data_filename
     end
   end
 
@@ -26,9 +48,26 @@ module CartoCSSHelper
       @size = size
     end
 
-    def generate
+    def open_file
       @data_file = open(Configuration.get_data_filename, 'w')
+    end
+
+    def close_file
+      @data_file.close
+    end
+
+    def prepare_file
+      open_file
       generate_prefix
+    end
+
+    def finish_file
+      generate_sufix
+      close_file
+    end
+
+    def generate
+      prepare_file
       if @type == 'node'
         generate_node_topology(@lat, @lon, @tags)
       elsif @type == 'way'
@@ -38,8 +77,7 @@ module CartoCSSHelper
       else
         raise 'this type of element does not exists'
       end
-      generate_sufix
-      @data_file.close
+      finish_file
     end
 
     def generate_node_topology(lat, lon, tags)

data/lib/cartocss_helper/downloader.rb

@@ -6,6 +6,12 @@ require 'sys/filesystem'
 
 module CartoCSSHelper
   class Downloader
+    def self.get_file_with_downloaded_osm_data_for_location(latitude, longitude, size)
+      query = get_query_to_download_data_around_location(latitude, longitude, size)
+      return get_overpass_query_results_file_location(query, "download data for #{latitude} #{longitude} (#{size})")
+    end
+
+
     def self.download_osm_data_for_location(latitude, longitude, size, accept_cache=true)
       filename = CartoCSSHelper::Configuration.get_path_to_folder_for_cache + "#{latitude} #{longitude} #{size}.osm"
       if File.exists?(filename)
@@ -15,7 +21,7 @@ module CartoCSSHelper
         File.delete(filename)
       end
       query = get_query_to_download_data_around_location(latitude, longitude, size)
-      text = get_overpass_query_results(query)
+      text = get_overpass_query_results(query, "download data for #{latitude} #{longitude} (#{size})")
       file = File.new(filename, 'w')
       file.write text
       file.close
@@ -23,12 +29,12 @@ module CartoCSSHelper
     end
 
     def self.get_query_to_download_data_around_location(latitude, longitude, size)
-      min_latitude = latitude - size/2
-      max_latitude = latitude + size/2
-      min_longitude = longitude - size/2
-      max_longitude = longitude + size/2
+      min_latitude = latitude - size.to_f/2
+      max_latitude = latitude + size.to_f/2
+      min_longitude = longitude - size.to_f/2
+      max_longitude = longitude + size.to_f/2
       bb = "#{min_latitude},#{min_longitude},#{max_latitude},#{max_longitude}"
-      query = '[timeout:3600];'
+      query = "[timeout:#{Downloader.get_allowed_timeout_in_seconds}];"
       query += "\n"
       query += "(node(#{bb});<;);"
       query += "\n"
@@ -44,18 +50,18 @@ module CartoCSSHelper
     end
 
     def self.locate_element_with_given_tags_and_type(tags, type, latitude, longitude)
-      max_range_in_km_for_radius = 100 #"Radius temporarly limited to 100 km to mitigate a bug.". Previously Overpass crashed for values larger than 3660
+      max_range_in_km_for_radius = 1600
 
       #special support for following tag values:  :any_value
       range = 10*1000
       loop do
-        list = Downloader.get_overpass_query_results(Downloader.get_query_to_get_location(tags, type, latitude, longitude, range))
+        list = Downloader.get_overpass_query_results(Downloader.get_query_to_get_location(tags, type, latitude, longitude, range), "find #{tags} within #{range/1000}km from #{latitude}, #{longitude}")
         if list.length != 0
           return self.list_returned_by_overpass_to_a_single_location(list)
         end
-        range=range+[range, 100000].min
+        range=range+[2*range, 200000].min
         if range >= max_range_in_km_for_radius*1000
-          list = Downloader.get_overpass_query_results(Downloader.get_query_to_get_location(tags, type, latitude, longitude, :infinity))
+          list = Downloader.get_overpass_query_results(Downloader.get_query_to_get_location(tags, type, latitude, longitude, :infinity), "find #{tags} across the world")
           if list.length != 0
             return self.list_returned_by_overpass_to_a_single_location(list)
           else
@@ -74,7 +80,7 @@ module CartoCSSHelper
 
     def self.get_query_to_get_location(tags, type, latitude, longitude, range)
       #special support for following tag values:  :any_value
-      locator = '[timeout:3600][out:csv(::lat,::lon;false)];'
+      locator = "[timeout:#{Downloader.get_allowed_timeout_in_seconds}][out:csv(::lat,::lon;false)];"
       locator += "\n"
       if type == 'closed_way'
         type = 'way'
@@ -119,22 +125,50 @@ module CartoCSSHelper
       return element
     end
 
-    def self.get_overpass_query_results(query, debug=false)
+    def self.get_overpass_query_results_file_location(query, description, debug=false)
+      filename = get_query_cache_filename(query)
+      get_overpass_query_results(query, description, debug)
+      return filename
+    end
+
+    def self.get_overpass_query_results(query, description, debug=false)
       cached = get_overpass_query_results_from_cache(query)
+      if cached == ''
+        raise OverpassRefusedResponse
+      end
       return cached unless cached == nil
 
       check_for_free_space
 
-      puts 'Running Overpass query'
+      puts 'Running Overpass query (connection initiated on ' + Time.now.to_s + ') ' + description
       if debug
         puts query
         puts
       end
-      cached = Downloader.run_overpass_query query
+      begin
+        cached = Downloader.run_overpass_query query, description
+      rescue OverpassRefusedResponse
+        write_to_cache(query, '')
+        raise OverpassRefusedResponse
+      end
+      write_to_cache(query, cached)
+      return cached
+    end
+
+    def self.write_to_cache(query, response)
       file = File.new(get_query_cache_filename(query), 'w')
-      file.write cached
+      file.write response
       file.close
-      return cached
+    end
+
+    def self.get_timestamp_of_file(timestamp_filename)
+      if !File.exists?(timestamp_filename)
+        return nil
+      end
+      f = File.new(timestamp_filename)
+      timestamp = f.mtime.to_i
+      f.close
+      return timestamp
     end
 
     def self.get_overpass_query_results_from_cache(query)
@@ -156,39 +190,88 @@ module CartoCSSHelper
     end
 
     def self.check_for_free_space
-      stat = Sys::Filesystem.stat(CartoCSSHelper::Configuration.get_path_to_folder_for_overpass_cache)
-      gb_available = stat.block_size * stat.blocks_available / 1024 / 1024 / 1024
-      if gb_available < 2
-        #TODO cleanup cache rather than crash
-        raise 'less than 2GB of free space on disk with cache folder'
+      if not_enough_free_space
+        attempt_cleanup
+        if not_enough_free_space
+          raise 'less than 2GB of free space on disk with cache folder'
+        end
+      end
+    end
+
+    def self.not_enough_free_space
+      minimum_gb = 2
+      return get_available_space_for_cache_in_gb < minimum_gb
+    end
+
+    def self.get_available_space_for_cache_in_gb
+      stat = Sys::Filesystem.stat(CartoCSSHelper::Configuration.get_path_to_folder_for_cache)
+      return stat.block_size * stat.blocks_available / 1024 / 1024 / 1024
+    end
+
+    def self.attempt_cleanup
+      if not_enough_free_space
+        delete_large_overpass_caches
       end
     end
 
-    def self.run_overpass_query(query, retry_count=0, retry_max=5)
-      #default timeout is set to 180: http://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
-      #[timeout:3600]; should be used in all queries executed here
+    def self.delete_file(file)
+      open(CartoCSSHelper::Configuration.get_path_to_folder_for_cache+'log.txt', 'a') { |file|
+        file.puts("deleting #{file}, #{File.size(file)/1024/1024}MB")
+      }
+      File.delete(file)
+    end
+
+    def self.delete_large_overpass_caches
+      #todo - find library that deals with caches like this, bug here may be unfunny
+      Dir.glob(CartoCSSHelper::Configuration.get_path_to_folder_for_overpass_cache+'*') {|file|
+        if File.size(file) > (1024 * 1024 * 50)
+          delete_file(file)
+        end
+      }
+    end
+
+    class OverpassRefusedResponse < IOError; end
+
+    def self.run_overpass_query(query, description, retry_count=0, retry_max=5)
       start = Time.now.to_s
       begin
-        return RestClient::Request.execute(:method => :get, :url => Downloader.format_query_into_url(query), :timeout => 3600)
-      rescue RestClient::RequestFailed => e
+        url = Downloader.format_query_into_url(query)
+        timeout = Downloader.get_allowed_timeout_in_seconds+10
+        return RestClient::Request.execute(:method => :get, :url => url, :timeout => timeout)
+      rescue RestClient::RequestTimeout
+        puts 'Overpass API refused to process this request. It will be not attemped again, most likely query is too complex.'
+        raise OverpassRefusedResponse
+      rescue RestClient::RequestFailed, RestClient::ServerBrokeConnection => e
         puts query
         puts e.response
         puts e.http_code
         puts start
-        puts Time.now.to_s
+        puts "Rerunning #{description} started at #{Time.now.to_s} (#{retry_count}/#{retry_max}) after #{e}"
         if retry_count < retry_max
           sleep 60*5
-          Downloader.run_overpass_query(query, retry_count+1, retry_max)
+          Downloader.run_overpass_query(query, description, retry_count+1, retry_max)
         else
           e.raise
         end
       rescue ArgumentError => e
         puts 'ArgumentError from rest-client, most likely caused by https://github.com/rest-client/rest-client/issues/359'
         puts 'try overpass query that will return smaller amount of data'
+        puts e
+        e.raise
+      rescue => e
+        puts 'query failed'
+        puts query
+        puts
+        puts url
+        puts e
         e.raise
       end
     end
 
+    def self.get_allowed_timeout_in_seconds
+      return 10 * 60
+    end
+
     def self.format_query_into_url(query)
       query = query.gsub(/\n/, '')
       query = query.gsub(/\t/, '')

data/lib/cartocss_helper/git.rb

@@ -9,18 +9,24 @@ module CartoCSSHelper
       end
       Dir.chdir(Configuration.get_path_to_tilemill_project_folder) {
         require 'open3'
-        if !system("git checkout #{branch} #{silence}")
-          raise 'failed checkout'
-        end
+        command = "git checkout #{branch} #{silence}"
+        Open3.popen3(command) {|_, stdout, stderr, wait_thr |
+          error = stderr.read.chomp
+          if error != '' or wait_thr.value.success? != true
+            raise 'failed checkout to ' + branch + ' due to ' + error
+          end
+          return
+        }
+        raise 'impossible happened'
       }
     end
 
     def get_commit_hash
       Dir.chdir(Configuration.get_path_to_tilemill_project_folder) {
-        Open3.popen3('git log -n 1 --pretty=format:"%H"') {|_, stdout, stderr, _|
+        Open3.popen3('git log -n 1 --pretty=format:"%H"') {|_, stdout, stderr, wait_thr|
           commit = stdout.read.chomp
           error = stderr.read.chomp
-          if error != ''
+          if error != '' or wait_thr.value.success? != true
             raise error
           end
           return commit

data/lib/cartocss_helper/image_generator.rb

@@ -2,6 +2,7 @@
 require_relative 'configuration.rb'
 require_relative 'heuristic.rb'
 require_relative 'data_file_handling.rb'
+require_relative 'tilemill_handler.rb'
 require 'fileutils'
 include CartoCSSHelper::Configuration
 include CartoCSSHelper::Heuristic
@@ -86,36 +87,7 @@ module CartoCSSHelper
     def generate_image(lat, lon, debug)
       export_filename = self.get_filename
       bbox_size = self.get_bbox_size
-      Scene.run_tilemill_export_image(lat, lon,  @zlevel, bbox_size, 200, export_filename, debug)
-    end
-
-    def self.run_tilemill_export_image(lat, lon, zlevel, bbox_size, image_size, export_filename, debug=false)
-      if File.exists?(export_filename)
-        if debug
-          puts 'wanted file exists'
-        end
-        return
-      end
-      silence = '> /dev/null 2>&1'
-      if debug
-        silence = ''
-      end
-      #--bbox=[xmin,ymin,xmax,ymax]
-      bbox = "#{lon-bbox_size/2},#{lat-bbox_size/2},#{lon+bbox_size/2},#{lat+bbox_size/2}"
-      params = "--format=png --width=#{image_size} --height=#{image_size} --static_zoom=#{zlevel} --bbox=\"#{bbox}\""
-      project_name = CartoCSSHelper::Configuration.get_tilemill_project_name
-      command = "node /usr/share/tilemill/index.js export #{project_name} '#{export_filename}' #{params} #{silence}"
-      if debug
-        puts command
-      end
-      system command
-      unless File.exists?(export_filename)
-        if !debug
-          puts 'rerunning failed image generation with enabled debug'
-          return self.run_tilemill_export_image(lat, lon, zlevel, bbox_size, image_size, export_filename, true)
-        end
-        raise 'generation of file ' + export_filename + ' failed'
-      end
+      TilemillHandler.run_tilemill_export_image(lat, lon,  @zlevel, bbox_size, 200, export_filename, debug)
     end
   end
 end

data/lib/cartocss_helper/style_specific/default_osm_style.rb

@@ -44,6 +44,7 @@ module CartoCSSHelper
             TagRenderingStatus.new('amenity', 'atm', :primary),
             TagRenderingStatus.new('amenity', 'bank', :primary),
             TagRenderingStatus.new('amenity', 'bar', :primary),
+            TagRenderingStatus.new('amenity', 'bench', :primary),
             TagRenderingStatus.new('amenity', 'bicycle_parking', :primary),
             TagRenderingStatus.new('amenity', 'bicycle_rental', :primary),
             TagRenderingStatus.new('amenity', 'biergarten', :primary),
@@ -66,6 +67,7 @@ module CartoCSSHelper
             TagRenderingStatus.new('amenity', 'grave_yard', :primary),
             TagRenderingStatus.new('amenity', 'hospital', :primary),
             TagRenderingStatus.new('amenity', 'hunting_stand', :primary),
+            TagRenderingStatus.new('amenity', 'ice_cream', :primary),
             TagRenderingStatus.new('amenity', 'kindergarten', :primary),
             TagRenderingStatus.new('amenity', 'library', :primary),
             TagRenderingStatus.new('amenity', 'parking', :primary),
@@ -85,6 +87,7 @@ module CartoCSSHelper
             TagRenderingStatus.new('amenity', 'toilets', :primary),
             TagRenderingStatus.new('amenity', 'townhall', :primary),
             TagRenderingStatus.new('amenity', 'university', :primary),
+            TagRenderingStatus.new('amenity', 'waste_basket', :primary),
             TagRenderingStatus.new('area', 'no', :composite, {'barrier' => 'hedge'}),
             TagRenderingStatus.new('area', 'yes', :composite, {'barrier' => 'hedge'}),
             TagRenderingStatus.new('barrier', '*', :primary),
@@ -171,9 +174,9 @@ module CartoCSSHelper
             TagRenderingStatus.new('highway', 'turning_loop', :primary), #note: special topology is required
             TagRenderingStatus.new('highway', 'unclassified', :primary),
             TagRenderingStatus.new('historic', 'archaeological_site', :primary),
-            TagRenderingStatus.new('historic', 'castle_walls', :primary),
             TagRenderingStatus.new('historic', 'citywalls', :primary),
             TagRenderingStatus.new('historic', 'memorial', :primary),
+            TagRenderingStatus.new('historic', 'monument', :primary),
             TagRenderingStatus.new('horse', 'designated', :composite, {'highway'=>'path'}),
             TagRenderingStatus.new('intermittent', 'yes', :composite, {'waterway'=>'river'}),
             TagRenderingStatus.new('junction', 'yes', :composite, {'name'=>'a'}),
@@ -191,6 +194,7 @@ module CartoCSSHelper
             TagRenderingStatus.new('landuse', 'forest', :primary),
             TagRenderingStatus.new('landuse', 'garages', :primary),
             TagRenderingStatus.new('landuse', 'grass', :primary),
+            TagRenderingStatus.new('landuse', 'greenhouse_horticulture', :primary),
             TagRenderingStatus.new('landuse', 'industrial', :primary),
             TagRenderingStatus.new('landuse', 'landfill', :primary),
             TagRenderingStatus.new('landuse', 'meadow', :primary),
@@ -525,6 +529,13 @@ module CartoCSSHelper
             TagRenderingStatus.new('waterway', 'stream', :primary),
             TagRenderingStatus.new('waterway', 'wadi', :primary),
             TagRenderingStatus.new('waterway', 'weir', :primary),
+            TagRenderingStatus.new('wetland', 'bog', :composite, {'natural' => 'wetland'}),
+            TagRenderingStatus.new('wetland', 'marsh', :composite, {'natural' => 'wetland'}),
+            TagRenderingStatus.new('wetland', 'mud', :composite, {'natural' => 'wetland'}),
+            TagRenderingStatus.new('wetland', 'string_bog', :composite, {'natural' => 'wetland'}),
+            TagRenderingStatus.new('wetland', 'swamp', :composite, {'natural' => 'wetland'}),
+            TagRenderingStatus.new('wetland', 'tidalflat', :composite, {'natural' => 'wetland'}),
+            TagRenderingStatus.new('wetland', 'wet_meadow', :composite, {'natural' => 'wetland'}),
         ]
     end
 
@@ -547,6 +558,7 @@ module CartoCSSHelper
           {'highway' => 'track'}, #tracktype=grade1...
           {'amenity' => 'parking'}, #access=*...
           #{'barrier' => 'hedge'}, #area=yes
+          {'natural' => 'wetland'}, #wetland=bog...
       ]
     end
   end

data/lib/cartocss_helper/tilemill_handler.rb

@@ -0,0 +1,36 @@
+module CartoCSSHelper
+  class TilemillHandler
+    def self.run_tilemill_export_image(lat, lon, zlevel, bbox_size, image_size, export_filename, debug=false)
+      if File.exists?(export_filename)
+        if debug
+          puts 'wanted file exists'
+        end
+        return
+      end
+      silence = '> /dev/null 2>&1'
+      if debug
+        silence = ''
+      end
+      #--bbox=[xmin,ymin,xmax,ymax]
+      xmin = lon-bbox_size/2
+      ymin = lat-bbox_size/2
+      xmax = lon+bbox_size/2
+      ymax = lat+bbox_size/2
+      bbox = "#{xmin},#{ymin},#{xmax},#{ymax}"
+      params = "--format=png --width=#{image_size} --height=#{image_size} --static_zoom=#{zlevel} --bbox=\"#{bbox}\""
+      project_name = CartoCSSHelper::Configuration.get_tilemill_project_name
+      command = "node /usr/share/tilemill/index.js export #{project_name} '#{export_filename}' #{params} #{silence}"
+      if debug
+        puts command
+      end
+      system command
+      unless File.exists?(export_filename)
+        if !debug
+          puts 'rerunning failed image generation with enabled debug'
+          return self.run_tilemill_export_image(lat, lon, zlevel, bbox_size, image_size, export_filename, true)
+        end
+        raise 'generation of file ' + export_filename + ' failed'
+      end
+    end
+  end
+end

data/lib/cartocss_helper/util/filehelper.rb

@@ -0,0 +1,6 @@
+# encoding: UTF-8
+module FileHelper
+  def self.make_string_usable_as_filename(string)
+    return string.gsub(/[\x00\/\\:\*\?\"<>\|]/, '_')
+  end
+end

data/lib/cartocss_helper/visualise_changes_diff_from_images.rb

@@ -34,7 +34,7 @@ module CartoCSSHelper
       @compared = compress(before, after)
 
       @image_size = image_size
-      @margin = 20
+      @margin = 10
       @standard_pointsize = 10
       @header_space = @standard_pointsize*1.5
       @diff_note_space = @standard_pointsize

data/lib/cartocss_helper/visualise_changes_image_generation.rb

@@ -1,10 +1,89 @@
 require_relative 'visualise_changes_diff_from_images'
 require_relative 'git'
 require_relative 'downloader'
+require_relative 'util/filehelper'
 
 module CartoCSSHelper
   class VisualDiff
-    def self.visualise_changes_synthethic_test(tags, type, on_water, zlevel_range, old_branch, new_branch)
+    @@job_pooling = false
+    @@jobs = []
+
+    def self.enable_job_pooling
+      #it results in avoiding loading the same database mutiple times
+      #useful if the same database will be used multiple times (for example the same place in multiple comparisons)
+      #use run_jobs function to run jobs
+      @@job_pooling = true
+    end
+
+    def self.disable_job_pooling
+      @@job_pooling = false
+    end
+
+    class MapGenerationJob
+      attr_reader :filename, :active
+      def initialize(filename, latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size)
+        @filename = filename
+        @latitude = latitude
+        @longitude = longitude
+        @zlevels = zlevels
+        @header = header
+        @old_branch = old_branch
+        @new_branch = new_branch
+        @download_bbox_size = download_bbox_size
+        @image_size = image_size
+        @active = true
+      end
+      def run_job
+        if !@active
+          return
+        end
+        print
+        source = CartoCSSHelper::VisualDiff::FileDataSource.new(@latitude, @longitude, @download_bbox_size, @filename)
+        CartoCSSHelper::VisualDiff.visualise_changes_for_given_source(@latitude, @longitude, @zlevels, @header, @new_branch, @old_branch, @image_size, source)
+        @active = false
+      end
+      def print
+        puts "#{@filename.gsub(Configuration.get_path_to_folder_for_cache, '#')} [#{@latitude};#{@longitude}], z: #{@zlevels}, text: #{@header}, '#{@old_branch}'->'#{@new_branch}', bbox:#{@download_bbox_size}, #{@image_size}px"
+      end
+    end
+
+    def self.add_job(filename, latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size)
+      print 'pool <- '
+      new_job = MapGenerationJob.new(filename, latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size)
+      new_job.print
+
+      raise "#{filename} does not exists" unless File.exists?(filename)
+      raise "#{latitude} is not a number" unless latitude.kind_of? Numeric
+      raise "#{longitude} is not a number" unless longitude.kind_of? Numeric
+      raise "#{zlevels} is not a range" unless zlevels.class == Range
+      raise "#{header} is not a string" unless header.class == String
+      raise "#{new_branch} is not a string" unless new_branch.class == String
+      raise "#{old_branch} is not a string" unless old_branch.class == String
+      raise "#{download_bbox_size} is not a number" unless download_bbox_size.kind_of? Numeric
+      raise "#{image_size} is not a integer" unless image_size.kind_of? Integer
+
+      @@jobs.push(new_job)
+    end
+
+    def self.run_jobs
+      for i in 0..@@jobs.length-1
+        if @@jobs[i].active
+          @@jobs[i].run_job
+          for x in 0..@@jobs.length-1
+            if @@jobs[i].filename == @@jobs[x].filename
+              @@jobs[x].run_job
+            end
+          end
+        end
+      end
+      @@jobs = []
+    end
+
+    def self.shuffle_jobs
+      @@jobs.shuffle!
+    end
+
+    def self.visualise_changes_synthethic_test(tags, type, on_water, zlevel_range, new_branch, old_branch)
       tags = VisualDiff.remove_magic_from_tags(tags)
       on_water_string = ''
       if on_water
@@ -16,7 +95,7 @@ module CartoCSSHelper
       old = VisualDiff.collect_images_for_synthethic_test(tags, type, on_water, zlevel_range)
       Git.checkout(new_branch)
       new = VisualDiff.collect_images_for_synthethic_test(tags, type, on_water, zlevel_range)
-      VisualDiff.pack_image_sets old, new, header, old_branch, new_branch, 200
+      VisualDiff.pack_image_sets old, new, header, new_branch, old_branch, 200
     end
 
     def self.remove_magic_from_tags(tags)
@@ -39,20 +118,18 @@ module CartoCSSHelper
       return collection
     end
 
-    class RealDataSource
-      def initialize(latitude, longitude, wanted_download_bbox_size)
-        wanted_download_bbox_size = [wanted_download_bbox_size, 0.001].max
-        wanted_download_bbox_size = [wanted_download_bbox_size, 0.400].min #0.5 is too big for Utrecht
-        @download_bbox_size = wanted_download_bbox_size
+    class FileDataSource
+      attr_reader :download_bbox_size, :data_filename
+      def initialize(latitude, longitude, download_bbox_size, filename)
+        @download_bbox_size = download_bbox_size
         @latitude = latitude
         @longitude = longitude
-        @data_filename = Downloader.download_osm_data_for_location(@latitude, @longitude, @download_bbox_size)
+        @data_filename = filename
         @loaded = false
       end
 
       def load
         if !@loaded
-          puts "\tloading data into database"
           DataFileLoader.load_data_into_database(@data_filename)
           puts "\tgenerating images"
           @loaded = true
@@ -60,28 +137,48 @@ module CartoCSSHelper
       end
 
       def get_timestamp
-        f = File.new(@data_filename)
-        return f.mtime.to_i
+        return Downloader.get_timestamp_of_file(@data_filename)
       end
     end
 
-    def self.visualise_changes_on_real_data(tags, type, wanted_latitude, wanted_longitude, zlevels, old_branch, new_branch)
+    def self.visualise_changes_on_real_data(tags, type, wanted_latitude, wanted_longitude, zlevels, new_branch, old_branch='master')
       #special support for following tag values:  :any_value
       header = "#{ VisualDiff.dict_to_pretty_tag_list(tags) } #{type} #{ wanted_latitude } #{ wanted_longitude } #{zlevels}"
       puts "visualise_changes_on_real_data <#{header}> #{old_branch} -> #{new_branch}"
-      latitude, longitude = Downloader.locate_element_with_given_tags_and_type tags, type, wanted_latitude, wanted_longitude
-      download_bbox_size = VisualDiff.scale_inverse zlevels.min, 0.03, 14
-      source = RealDataSource.new(latitude, longitude, download_bbox_size)
-      Git.checkout old_branch
-      old = VisualDiff.collect_images_for_real_data_test(tags, type, latitude, longitude, zlevels, source)
-      Git.checkout new_branch
-      new = VisualDiff.collect_images_for_real_data_test(tags, type, latitude, longitude, zlevels, source)
+      begin
+        latitude, longitude = Downloader.locate_element_with_given_tags_and_type tags, type, wanted_latitude, wanted_longitude
+      rescue Downloader::OverpassRefusedResponse
+        puts 'No nearby instances of tags and tag is not extremely rare - no generation of nearby location and wordwide search was impossible. No diff image will be generated for this location.'
+        return false
+      end
       header = "#{ VisualDiff.dict_to_pretty_tag_list(tags) } #{type} #{ wanted_latitude } #{ wanted_longitude } #{zlevels}"
-      VisualDiff.pack_image_sets old, new, header, old_branch, new_branch, 400
+      download_bbox_size = 0.4
+      visualise_changes_for_location(latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size)
+      return true
     end
 
-    def self.scale_inverse(zlevel, reference_value, reference_zlevel)
-      return VisualDiff.scale reference_zlevel, reference_value, zlevel
+    def self.visualise_changes_for_location(latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size=400)
+      filename = Downloader.get_file_with_downloaded_osm_data_for_location(latitude, longitude, download_bbox_size)
+      visualise_changes_for_location_from_file(filename, latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size)
+    end
+
+    def self.visualise_changes_for_location_from_file(filename, latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size=400)
+      prefix = ''
+      if @@job_pooling
+        prefix = 'pool <- '
+      end
+      add_job(filename, latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size)
+      if !@@job_pooling
+        run_jobs
+      end
+    end
+
+    def self.visualise_changes_for_given_source(latitude, longitude, zlevels, header, new_branch, old_branch, image_size, source)
+      Git.checkout old_branch
+      old = VisualDiff.collect_images_for_real_data_test(latitude, longitude, zlevels, source, image_size)
+      Git.checkout new_branch
+      new = VisualDiff.collect_images_for_real_data_test(latitude, longitude, zlevels, source, image_size)
+      VisualDiff.pack_image_sets old, new, header, new_branch, old_branch, image_size
     end
 
     def self.scale(zlevel, reference_value, reference_zlevel)
@@ -90,7 +187,7 @@ module CartoCSSHelper
       return reference_value*rescaler
     end
 
-    def self.collect_images_for_real_data_test(tags, type, latitude, longitude, zlevels, source)
+    def self.collect_images_for_real_data_test(latitude, longitude, zlevels, source, wanted_image_size=400)
       collection = []
       zlevels.each { |zlevel|
         image_size_for_16_zoom_level = 1000
@@ -98,7 +195,6 @@ module CartoCSSHelper
         mutiplier = 1000
         image_size = (image_size*mutiplier).to_int
         render_bbox_size = 0.015
-        wanted_image_size = 400
         ratio = 1.0*image_size/(wanted_image_size*mutiplier)
         image_size /= ratio
         render_bbox_size /= ratio
@@ -112,7 +208,8 @@ module CartoCSSHelper
           puts ratio
           raise "#{image_size} mismatches #{wanted_image_size}"
         end
-        filename = "#{CartoCSSHelper::Configuration.get_path_to_folder_for_branch_specific_cache+"#{tags.to_a.to_s} #{type} #{latitude} #{longitude} #{zlevel}zlevel #{image_size}px #{source.get_timestamp}.png"}"
+        cache_folder = CartoCSSHelper::Configuration.get_path_to_folder_for_branch_specific_cache
+        filename = "#{cache_folder+"#{latitude} #{longitude} #{zlevel}zlevel #{image_size}px #{source.get_timestamp} #{source.download_bbox_size}.png"}"
         if !File.exists?(filename)
           source.load
           Scene.run_tilemill_export_image(latitude, longitude, zlevel, render_bbox_size, image_size, filename)
@@ -122,14 +219,16 @@ module CartoCSSHelper
       return collection
     end
 
-    def self.pack_image_sets(old, new, header, old_branch, new_branch, image_size)
-      old_branch = old_branch.gsub(/[\x00\/\\:\*\?\"<>\|]/, '_')
-      new_branch = new_branch.gsub(/[\x00\/\\:\*\?\"<>\|]/, '_')
+    def self.pack_image_sets(old, new, header, new_branch, old_branch, image_size)
+      old_branch = FileHelper::make_string_usable_as_filename(old_branch)
+      new_branch = FileHelper::make_string_usable_as_filename(new_branch)
+      header_for_filename = FileHelper::make_string_usable_as_filename(header)
       filename_sufix = "#{ old_branch } -> #{ new_branch }"
-      filename = CartoCSSHelper::Configuration.get_path_to_folder_for_output + "#{header} #{filename_sufix}.png"
+      filename = CartoCSSHelper::Configuration.get_path_to_folder_for_output + "#{header_for_filename} #{filename_sufix} #{image_size}px.png"
       diff = FullSetOfComparedImages.new(old, new, header, filename, image_size)
       diff.save
     end
+
     def self.dict_to_pretty_tag_list(dict)
       result = ''
       dict.to_a.each { |tag|

data/lib/data/testing_locations.rb

@@ -0,0 +1,242 @@
+def get_list_of_testing_locations
+  return [
+      [50.1, 19.9], #Krakow
+      [53.2, -1.8], #rural uk
+      [36.1, 140.7], #Japan
+      [54.8, 31.7], #Russia
+      [21.3, 39.5], #Mecca
+      [41.4, 44.5], #Georgia
+      [51.5, -0.1], #London
+      #TODO: solve problems with response too big to store in memory
+      #[52.09, 5.11], #Utrecht
+
+      #http://overpass-turbo.eu/s/ar4
+      #capital locations
+      [-41.2887639, 174.7772239],
+      [37.9841493, 23.7279843],
+      [46.0498650, 14.5068921],
+      [55.6867243, 12.5700724],
+      [28.6138967, 77.2159562],
+      [33.6945756, 73.0643744],
+      [48.2083537, 16.3725042],
+      #[48.8565056, 2.3521334], TODO: solve problems with rest-client crashing and blabbing about negative integers
+      [46.9482713, 7.4514512],
+      [45.4210328, -75.6900219],
+      [59.9132694, 10.7391112],
+      [33.3024248, 44.3787992],
+      [40.4167047, -3.7035825],
+      [-35.2819367, 149.1288940],
+      [34.5197058, 69.1624302],
+      [39.9059093, 116.3913489],
+      [-25.7250000, 28.1847222],
+      [39.9220794, 32.8537580],
+      [59.3251172, 18.0710935],
+      [35.6861720, 51.4223519],
+      [50.4501071, 30.5240501],
+      [53.9023386, 27.5619023],
+      [23.1379911, -82.3658562],
+      [56.9493977, 24.1051846],
+      [-4.3217055, 15.3125974],
+      [54.6843135, 25.2853984],
+      [55.1598408, 61.4025547],
+      [42.8766343, 74.6070116],
+      [41.3123363, 69.2787079],
+      [38.5762757, 68.7863940],
+      [47.9184676, 106.9177016],
+      [51.1282789, 71.4305478],
+      [13.5248339, 2.1098226],
+      [-8.8271656, 13.2436665],
+      [12.6500083, -8.0000014],
+      [36.8001077, 10.1847938],
+      [6.3280343, -10.7977882],
+      [4.3909085, 18.5505422],
+      [-15.4166967, 28.2813812],
+      [12.3681481, -1.5270853],
+      [12.1191543, 15.0502758],
+      [14.6930042, -17.4470260],
+      [8.4790017, -13.2680158],
+      [9.5171253, -13.6999235],
+      [0.3900022, 9.4540009],
+      [9.0107934, 38.7612525],
+      [5.5600141, -0.2057437],
+      [-25.9662133, 32.5674498],
+      [11.5936649, 43.1472403],
+      [-1.2832533, 36.8172449],
+      [-18.9100122, 47.5255809],
+      [15.5933247, 32.5356502],
+      [14.9160169, -23.5096132],
+      [62.0120000, -6.7680000],
+      [31.7774287, 35.2247239],
+      [59.4372155, 24.7453688],
+      [-29.3100536, 27.4782219],
+      [-24.6585857, 25.9084905],
+      [39.0196396, 125.7525841],
+      [36.5658725, 53.0585690],
+      [-33.9289049, 18.4172485],
+      [-22.5744184, 17.0791233],
+      [-15.7934036, -47.8823172],
+      [2.0427779, 45.3385636],
+      [26.2235041, 50.5822436],
+      [-33.4377968, -70.6504451],
+      [12.5268736, -70.0356845],
+      [12.1091242, -68.9316546],
+      [12.1471741, -68.2740783],
+      [6.9342870, 79.8532704],
+      [49.6112768, 6.1297990],
+      [18.4652988, -66.1166657],
+      [17.9712148, -76.7928128],
+      [-0.2201687, -78.5120913],
+      [42.5069391, 1.5212467],
+      [-12.0458271, -77.0304845],
+      [-1.9508511, 30.0615075],
+      [44.8178787, 20.4568089],
+      [-3.3638125, 29.3675028],
+      [27.7087963, 85.3202438],
+      [41.8933439, 12.4830718],
+      [21.0292095, 105.8524700],
+      [-34.6128690, -58.4459789],
+      [47.4983815, 19.0404707],
+      [-6.1791181, 35.7468174],
+      [43.4930746, 43.6193570],
+      [44.4361390, 26.1027436],
+      [4.8472017, 31.5951655],
+      [-17.7414972, 168.3150163],
+      [25.0783456, -77.3383331],
+      [17.9640988, 102.6133707],
+      [42.4415238, 19.2621081],
+      [47.0122737, 28.8605936],
+      [40.1776121, 44.5125849],
+      [38.8949549, -77.0366456],
+      [41.6509502, 41.6360085],
+      [41.9960424, 21.4316707],
+      [12.1461244, -86.2737170],
+      [17.2960919, -62.7223010],
+      [9.9325612, -84.0795739],
+      [13.6977587, -89.1930100],
+      [14.5906216, 120.9799696],
+      [23.5997857, 58.5451305],
+      [-9.4797487, 147.1503589],
+      [15.2984622, -61.3875049],
+      [52.5170365, 13.3888599],
+      [50.2796448, 57.2124606],
+      [-20.1637281, 57.5045331],
+      [-26.3257447, 31.1446630],
+      [39.5089040, 54.3631062],
+      [12.0535331, -61.7518050],
+      [13.1561653, -61.2248777],
+      [14.0095719, -60.9903195],
+      [13.1018264, -59.6188480],
+      [18.0794011, -15.9780393],
+      [13.4553495, -16.5756457],
+      [-25.2960638, -57.6311446],
+      [11.8613244, -15.5830554],
+      [15.3421010, 44.2005197],
+      [24.4747961, 54.3705762],
+      [-8.5536809, 125.5784093],
+      [15.3389974, 38.9326725],
+      [35.6823815, 139.7530053],
+      [18.0250713, -63.0483073],
+      [38.7130574, -9.1380056],
+      [-21.2074736, -159.7708145],
+      [52.3710088, 4.9001115],
+      [-7.7441461, 113.2158401],
+      [30.0488185, 31.2436663],
+      [48.4814020, 135.0769400],
+      [18.3411365, -64.9327890],
+      [43.9363996, 12.4466991],
+      [5.8216198, -55.1771974],
+      [17.2502830, -88.7694263],
+      [-17.8317726, 31.0456859],
+      [18.5473270, -72.3395928],
+      [33.8959203, 35.4784300],
+      [3.7528278, 8.7800610],
+      [6.8025766, -58.1628612],
+      [0.3389242, 6.7313031],
+      [-34.9059039, -56.1913569],
+      [21.4607723, -71.1399956],
+      [34.0223602, -6.8390357],
+      [14.0931919, -87.2012631],
+      [-19.0478620, -65.2596023],
+      [18.4801972, -69.9421110],
+      [24.6319692, 46.7150648],
+      [19.7540045, 96.1344976],
+      [6.8091068, -5.2732628],
+      [3.8689867, 11.5213344],
+      [-11.6931255, 43.2543044],
+      [7.0909924, 171.3816354],
+      [-13.8343691, -171.7692793],
+      [41.9034912, 12.4528349],
+      [17.1184569, -61.8448509],
+      [52.2319237, 21.0067265],
+      [-4.2694407, 15.2712256],
+      [-19.0534159, -169.9191992],
+      [14.6417889, -90.5132239],
+      [17.1393977, -62.6216154],
+      [48.1535383, 17.1096711],
+      [1.2904527, 103.8520380],
+      [29.3797091, 47.9735629],
+      [11.5682710, 104.9224426],
+      [-8.5211767, 179.1976747],
+      [4.5980478, -74.0760867],
+      [16.7053047, -62.2130447],
+      [1.3497964, 173.0277185],
+      [6.9207440, 158.1627143],
+      [7.4966382, 134.6349089],
+      [0.3177137, 32.5813539],
+      [19.2869856, -81.3717001],
+      [-9.4312971, 159.9552773],
+      [-2.2100307, 113.9267408],
+      [64.1750290, -51.7355386],
+      [-21.1343401, -175.2018085],
+      [33.5130695, 36.3095814],
+      [25.0375167, 121.5637000],
+      [8.9500000, 38.7666700],
+      [8.9707433, -79.5344539],
+      [60.1666277, 24.9435079],
+      [27.4730932, 89.6296788],
+      [35.8987546, 14.5134889],
+      [45.8131545, 15.9770298],
+      [50.0874401, 14.4212556],
+      [41.6934350, 44.8014979],
+      [13.7529438, 100.4941219],
+      [50.8465565, 4.3516970],
+      [31.9515694, 35.9239625],
+      [6.4990718, 2.6253361],
+      [25.2856329, 51.5264162],
+      [55.7516335, 37.6187042],
+      [4.8895453, 114.9417574],
+      [32.8966720, 13.1777923],
+      [42.6977211, 23.3225964],
+      [43.7311966, 7.4196400],
+      [37.9396678, 58.3874263],
+      [11.1852893, -60.7350240],
+      [41.3279457, 19.8185323],
+      [3.1570976, 101.7009528],
+      [35.1739302, 33.3647260],
+      [37.5666791, 126.9782914],
+      [47.1392862, 9.5227962],
+      [-14.2753717, -170.7048530],
+      [15.1759138, 145.7432916],
+      [18.2108070, -63.0535045],
+      [26.3544056, -9.5741765],
+      [43.8515421, 18.3886246],
+      [-2.9888297, 104.7568570],
+      [64.1459810, -21.9422367],
+      [40.3754289, 49.8328549],
+      [6.1258594, 1.2248504],
+      [54.1497740, -4.4779021],
+      [-18.1415884, 178.4421662],
+      [46.8516039, 29.6349435],
+      [38.3716205, 26.1342054],
+      [49.1855551, -2.1098279],
+      [10.6572142, -61.5180297],
+      [-1.6141280, 103.5796708],
+      [42.6636827, 21.1639418],
+      [-13.9734560, 33.7878122],
+      [43.7311424, 7.4197576],
+      [6.9031663, 79.9091644],
+      [-8.5202604, 179.1974823],
+      [4.1779879, 73.5107387],
+      [53.3497645, -6.2602732],
+  ]
+end

data/readme.md

@@ -1,6 +1,6 @@
 ##CartoCSS Helper
 
-It is a tool making development of CartoCSS styles more efficient. Automates actions necessary to produce test images and validates style. Loading data using osm2pgsql, rendering with TileMill, obtaining test data from overpass turbo.
+It is a tool making development of CartoCSS styles more efficient. Allows to write simple Ruby scripts that generate comparison images of how rendering changed between two versions of map style. It is also possible to generate validation reports detecting for example too low values of text-dy. Loading data using osm2pgsql, rendering with TileMill, obtaining test data from overpass turbo.
 
 Proper testing of map rendering requires testing covering mutiple situations. Doing it manually means preparing test data and locating places with applicable real data, loading map data into database, then moving viewport of TileMill to check every data location across multiple zoom levels. It is painfully slow.
 
@@ -8,17 +8,48 @@ Fortunately it is possible to automate it almost entirely. With this tool testin
 
 It is work in progress, major problems that should be solved include:
 
-* Lack of documentation
+* Improving documentation
 * Executing git checkout commands on map style git repository during normal script operation (https://github.com/matkoniecz/CartoCSSHelper/issues/9)
 * Current rendering method is obnoxiously slow. (https://github.com/matkoniecz/CartoCSSHelper/issues/1)
 
+###Installation
+
+Assumes that osm2pgsql and TileMill are installed.
+
+This tool is available as cartocss_helper Ruby gem.
+
+Install ruby - see https://www.ruby-lang.org/en/documentation/installation/ for details.
+
+Unfortunately, standard `gem install cartocss_helper` may not be enough as CartoCSS Helper depends on RMagick gem that requires special installation.
+
+Install RMagick gem. `gem install rmagick` may not be enough - in my case, on Ubuntu 14.04 it was necessary to start from installing additional packages (`apt-get install ruby-dev imagemagick libmagickcore-dev libmagickwand-dev` solved the problem), followed by `gem install rmagick`.
+
+Finally, run `gem install cartocss_helper`.
+
 ###Example
 
-`require cartocss_helper`
-`tags = {'landuse' => 'village_green', 'tourism' => 'attraction'}`
-`CartoCSSHelper.test tags, 'master', 'v2.28.1'`
+It is assumed that TileMill project with Default OSM Style (https://github.com/gravitystorm/openstreetmap-carto/) is located at `~/Documents/MapBox/project/osm-carto`.
+
+    require cartocss_helper
+    CartoCSSHelper::Configuration.set_style_specific_data(CartoCSSHelper::StyleDataForDefaultOSM.get_style_data)
+    CartoCSSHelper::Configuration.set_path_to_tilemill_project_folder(File.join(ENV['HOME'], 'Documents', 'MapBox', 'project', 'osm-carto', ''))
+    CartoCSSHelper::Configuration.set_path_to_folder_for_output(File.join(ENV['HOME'], 'Documents', 'CartoCSSHelper-output', ''))
+    CartoCSSHelper::Configuration.set_path_to_folder_for_cache(File.join(ENV['HOME'], 'Documents', 'CartoCSSHelper-tmp', ''))
+
+    tags = {'landuse' => 'village_green', 'tourism' => 'attraction'}
+    CartoCSSHelper.test tags, 'master', 'v2.28.1'
+
+First part of this Ruby script is responsible for loading gem and configuration, including location of cache folder and folder where produced comparison images will be saved.
+
+Second part runs quick test for specified tag combination rendering only this element as node, way and a closed way. It is followed by locating multiple places across globe where such tag combination is used.
+
+For each test case images are produced both for current `master` branch and release `v2.28.1` across multiple zoom levels. Finally tool generates before/after comparisons for each case. Some of generated images were used in https://github.com/gravitystorm/openstreetmap-carto/issues/1371.
+
+It is also possible to look for certain keys, with any value:
+
+    tags = {'landuse' => 'village_green', 'tourism' => 'attraction', 'name' => :any_value}
+    CartoCSSHelper.test tags, 'master', 'v2.28.1'
 
-Runs quick test for specified tag combination rendering only this element, followed by locating multiple places across globe where such tags are used. For each test case images are produced both for current `master` branch and release `v2.28.1` across multiple zoom levels. Finally tool generates before/after comparisons for each case. Some of generated images were used in https://github.com/gravitystorm/openstreetmap-carto/issues/1371.
 
 ###Automated stuff
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cartocss_helper
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 1.0.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-11 00:00:00.000000000 Z
+date: 2015-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -74,13 +74,16 @@ files:
 - lib/cartocss_helper/image_generator.rb
 - lib/cartocss_helper/validator.rb
 - lib/cartocss_helper/downloader.rb
+- lib/cartocss_helper/tilemill_handler.rb
 - lib/cartocss_helper/git.rb
+- lib/cartocss_helper/util/filehelper.rb
 - lib/cartocss_helper/heuristic.rb
 - lib/cartocss_helper/visualise_changes_diff_from_images.rb
 - lib/cartocss_helper/configuration.rb
 - lib/cartocss_helper/style_specific/style_specific.rb
 - lib/cartocss_helper/style_specific/default_osm_style.rb
 - lib/cartocss_helper/data_file_handling.rb
+- lib/data/testing_locations.rb
 - license.txt
 - readme.md
 homepage: https://github.com/matkoniecz/CartoCSSHelper