cartocss_helper 1.2.1 → 4.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8686f0a5acb7db504aec14f93f683c89dfd06248
+  data.tar.gz: 1f482eba9f389cdceb37e5253c2520a47725128b
+SHA512:
+  metadata.gz: f23d8415d318009faf84f70e3f6ae1fb63b80033227c340ba3d647c45a0cc86314c4ec76dc4e07e3d5a2a751415b0cb60affa2a6db92351f1cd031492af54a19
+  data.tar.gz: e86b7bca3b6b4f9f1a84aee7d861d121bb276b1667e7a005d86fc052e1bfe858266da79c2f256de19c3824802fbbcdcf4f33ce2ddd6889f758ff199a0c11d7db

data/lib/cartocss_helper.rb

@@ -5,150 +5,135 @@ module CartoCSSHelper
 end
 require_relative 'cartocss_helper/tag_lister.rb'
 require_relative 'cartocss_helper/visualise_changes_image_generation.rb'
-require_relative 'cartocss_helper/downloader.rb'
+require_relative 'cartocss_helper/overpass_query_generator.rb'
 require_relative 'cartocss_helper/data_file_handling.rb'
 require_relative 'cartocss_helper/validator.rb'
 require_relative 'cartocss_helper/git.rb'
+require_relative 'cartocss_helper/util/generic_downloader.rb'
+require_relative 'cartocss_helper/notes_downloader.rb'
+require_relative 'cartocss_helper/style_specific/default_osm_style.rb'
 require_relative 'data/testing_locations'
+
 include CartoCSSHelper::Validator
 include CartoCSSHelper::Git
 
 module CartoCSSHelper
-  def self.test_tag_on_real_data(tags, new_branch, old_branch, zlevels, types=['node', 'closed_way', 'way'], min = 4, skip = 0)
-    types.each {|type|
+  def self.test_tag_on_real_data(tags, new_branch, old_branch, zlevels, types = ['node', 'closed_way', 'way'], min = 4, skip = 0)
+    types.each do |type|
       test_tag_on_real_data_for_this_type(tags, new_branch, old_branch, zlevels, type, min, skip)
-    }
+    end
   end
 
   def self.test_tag_on_real_data_for_this_type(tags, new_branch, old_branch, zlevels, type, min = 4, skip = 0)
-    if type.kind_of?(Array)
-      type = type[0]
-    end
+    type = type[0] if type.is_a?(Array)
     generated = 0
 
     n = 0
-    max_n = get_maxn_for_nth_location
+    max_n = CartoCSSHelper.get_maxn_for_nth_location
     max_n -= skip
     skip_string = ''
-    if skip > 0
-      skip_string = " (#{skip} locations skipped)"
-    end
+    skip_string = " (#{skip} locations skipped)" if skip > 0
     while generated < min
-      location = get_nth_location(n + skip)
-      generated +=1 if CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, location[0], location[1], zlevels, new_branch, old_branch)
-      n+=1
-      if n > max_n
-        return
-      end
+      location = CartoCSSHelper.get_nth_location(n + skip)
+      generated += 1 if CartoCSSHelper::VisualDiff.visualise_on_overpass_data(tags, type, location[0], location[1], zlevels, new_branch, old_branch)
+      n += 1
+      return if n > max_n
       puts "#{n}/#{max_n} locations checked #{skip_string}. #{generated}/#{min} testing location found"
     end
   end
 
   def self.get_maxn_for_nth_location
-    return get_list_of_testing_locations.length-1
+    return get_list_of_testing_locations.length - 1
   end
 
   def self.get_nth_location(n)
     return get_list_of_testing_locations[n]
   end
 
-
   def self.add_common_secondary_tags(tags)
-    added_tags = {'name' => 'ÉÉÉÉÉÉ ÉÉÉÉÉÉ', 'ref' => '1', 'ele' => '8000', 'operator' => 'ÉÉ ÉÉ ÉÉ operator', 'brand' => 'ÉÉ ÉÉ ÉÉ brand'}
+    added_tags = { 'name' => 'ÉÉÉÉÉÉ ÉÉÉÉÉÉ', 'ref' => '1', 'ele' => '8000', 'operator' => 'ÉÉ ÉÉ ÉÉ operator', 'brand' => 'ÉÉ ÉÉ ÉÉ brand' }
     return tags.merge(added_tags)
   end
 
-  def self.test_tag_on_sythetic_data(tags, new_branch, old_branch='master', zlevels=Configuration.get_min_z..Configuration.get_max_z, types=['node', 'closed_way', 'way'], test_on_water=false)
+  def self.test_tag_on_sythetic_data(tags, new_branch, old_branch = 'master', zlevels = Configuration.get_min_z..Configuration.get_max_z, types = ['node', 'closed_way', 'way'], test_on_water = false)
     syn_tags = add_common_secondary_tags(tags)
-    types.each {|type|
-      CartoCSSHelper::VisualDiff.visualise_changes_synthethic_test(syn_tags, type, test_on_water, zlevels, new_branch, old_branch)
-    }
+    types.each do |type|
+      CartoCSSHelper::VisualDiff.visualise_on_synthethic_data(syn_tags, type, test_on_water, zlevels, new_branch, old_branch)
+    end
   end
 
-  def self.test(tags, new_branch, old_branch='master', zlevels=Configuration.get_min_z..Configuration.get_max_z, types=['node', 'closed_way', 'way'], test_on_water=false)
-    puts "processing #{VisualDiff.dict_to_pretty_tag_list(tags)}"
+  def self.test(tags, new_branch, old_branch = 'master', zlevels = Configuration.get_min_z..Configuration.get_max_z, types = ['node', 'closed_way', 'way'], test_on_water = false)
+    puts "processing #{VisualDiff.tag_dict_to_string(tags)}"
     test_tag_on_sythetic_data(tags, new_branch, old_branch, zlevels, types, test_on_water)
     test_tag_on_real_data(tags, new_branch, old_branch, zlevels, types)
   end
 
-  def self.probe(tags, new_branch, old_branch='master', zlevels=Configuration.get_min_z..Configuration.get_max_z, types=['node', 'closed_way', 'way'], test_on_water=false)
+  def self.probe(tags, new_branch, old_branch = 'master', zlevels = Configuration.get_min_z..Configuration.get_max_z, types = ['node', 'closed_way', 'way'], test_on_water = false)
     syn_tags = add_common_secondary_tags(tags)
-    types.each {|type|
-      CartoCSSHelper::VisualDiff.visualise_changes_synthethic_test(syn_tags, type, test_on_water, zlevels, new_branch, old_branch)
-    }
+    types.each do |type|
+      CartoCSSHelper::VisualDiff.visualise_on_synthethic_data(syn_tags, type, test_on_water, zlevels, new_branch, old_branch)
+    end
   end
 
   def self.get_latitude_longitude_from_url(url)
-    if(url.scan(/[\/]((-|)\d+(\.\d+))/)).length >= 2
-      latitude = url.scan(/[\/]((-|)\d+(\.\d+))/)[0][0].to_f
-      longitude = url.scan(/[\/]((-|)\d+(\.\d+))/)[1][0].to_f
-      return latitude, longitude
-    end
-    latitude = url.scan(/[\/=]((-|)\d+(\.\d+))/)[0][0].to_f
-    longitude = url.scan(/[\/=]((-|)\d+(\.\d+))/)[1][0].to_f
+    lat_lon = url.scan(/[\/=]((-|)\d+(\.\d+))/)
+    latitude = lat_lon[0][0].to_f
+    longitude = lat_lon[1][0].to_f
     return latitude, longitude
   end
 
-  def self.visualise_place_by_url(url, zlevels, new_branch, old_branch='master', header=nil, download_bbox_size=0.04, image_size = 350)
-    if header == nil
-      header = url
-    end
+  def self.visualise_place_by_url(url, zlevels, new_branch, old_branch = 'master', header = nil, download_bbox_size = 0.04, image_size = 350)
+    header = url if header == nil
 
     raise "#{url} is not a string, it is #{url.class}" unless url.class == String
     raise "#{zlevels} is not a range, it is #{zlevels.class}" unless zlevels.class == Range
     raise "#{new_branch} is not a string, it is #{new_branch.class}" unless new_branch.class == String
     raise "#{old_branch} is not a string, it is #{old_branch.class}" unless old_branch.class == String
     raise "#{header} is not a string, it is #{header.class}" unless header.class == String
-    raise "#{download_bbox_size} is not a number" unless download_bbox_size.kind_of? Numeric
-    raise "#{image_size} is not a integer" unless image_size.kind_of? Integer
+    raise "#{download_bbox_size} is not a number" unless download_bbox_size.is_a? Numeric
+    raise "#{image_size} is not a integer" unless image_size.is_a? Integer
 
     latitude, longitude = get_latitude_longitude_from_url(url)
-    header += ' ' + old_branch + '->' + new_branch + ' ' + zlevels.to_s + ' '+ image_size.to_s + 'px'
-    CartoCSSHelper::VisualDiff.visualise_changes_for_location(latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size)
+    header += ' ' + old_branch + '->' + new_branch + ' ' + zlevels.to_s + ' ' + image_size.to_s + 'px'
+    CartoCSSHelper::VisualDiff.visualise_for_location(latitude, longitude, zlevels, header, new_branch, old_branch, download_bbox_size, image_size)
   end
 
   def self.get_place_of_storage_of_resource_under_url(url)
-    return CartoCSSHelper::Configuration.get_path_to_folder_for_overpass_cache + '.manual.cache' + FileHelper::make_string_usable_as_filename(url)
+    return CartoCSSHelper::Configuration.get_path_to_folder_for_overpass_cache + '.manual.cache' + FileHelper.make_string_usable_as_filename(url)
   end
 
-  def self.download_remote_file(url)
+  def self.download_remote_file(url, clear_cache = false)
     filename = get_place_of_storage_of_resource_under_url(url)
-    if !File.exists?(filename)
-      begin
-        url = url
-        timeout = 600
-        data = RestClient::Request.execute(:method => :get, :url => url, :timeout => timeout)
-      rescue => e
-        puts "visualise_place_by_remote_file failed to fetch #{url}"
-        raise e
-      end
-      file = File.new(filename, 'w')
-      file.write data
-      file.close
+    if clear_cache
+      File.delete(filename) if File.exist?(filename)
+    end
+    unless File.exist?(filename)
+      url = url
+      timeout = 600
+      downloader = GenericCachedDownloader.new(timeout: timeout)
+      return downloader.get_specified_resource(url, filename)
     end
   end
 
-  def self.visualise_place_by_remote_file(url, latitude, longitude, zlevels, new_branch, old_branch='master', header=nil, bb=0.04, image_size = 350)
+  def self.visualise_place_by_remote_file(url, latitude, longitude, zlevels, new_branch, old_branch = 'master', header = nil, bb = 0.04, image_size = 350)
     download_remote_file(url)
     filename = get_place_of_storage_of_resource_under_url(url)
     visualise_place_by_file(filename, latitude, longitude, zlevels, new_branch, old_branch, header, bb, image_size)
   end
 
-  def self.visualise_place_by_file(filename, latitude, longitude, zlevels, new_branch, old_branch='master', header=nil, bb=0.04, image_size = 350)
-    raise "#{filename} does not exists" unless File.exists?(filename)
-    raise "#{latitude} is not a number" unless latitude.kind_of? Numeric
-    raise "#{longitude} is not a number" unless longitude.kind_of? Numeric
+  def self.visualise_place_by_file(filename, latitude, longitude, zlevels, new_branch, old_branch = 'master', header = nil, bb = 0.04, image_size = 350)
+    raise "#{filename} does not exists" unless File.exist?(filename)
+    raise "#{latitude} is not a number" unless latitude.is_a? Numeric
+    raise "#{longitude} is not a number" unless longitude.is_a? Numeric
     raise "#{zlevels} is not a range" unless zlevels.class == Range
     raise "#{new_branch} is not a string" unless new_branch.class == String
     raise "#{old_branch} is not a string" unless old_branch.class == String
     raise "#{header} is not a string" unless header.class == String
-    raise "#{bb} is not a number" unless bb.kind_of? Numeric
-    raise "#{image_size} is not a integer" unless image_size.kind_of? Integer
+    raise "#{bb} is not a number" unless bb.is_a? Numeric
+    raise "#{image_size} is not a integer" unless image_size.is_a? Integer
 
-    if header == nil
-      header = filename
-    end
+    header = filename if header == nil
     header += ' ' + old_branch + '->' + new_branch + '[' + latitude.to_s + ',' + longitude.to_s + ']' + ' ' + image_size.to_s + 'px'
-    CartoCSSHelper::VisualDiff.visualise_changes_for_location_from_file(filename, latitude, longitude, zlevels, header, new_branch, old_branch, bb, image_size)
+    CartoCSSHelper::VisualDiff.visualise_for_location_from_file(filename, latitude, longitude, zlevels, header, new_branch, old_branch, bb, image_size)
   end
-end
+end

data/lib/cartocss_helper/configuration.rb

@@ -16,88 +16,135 @@ module CartoCSSHelper::Configuration
     return @style_specific_data
   end
 
-	def get_max_z
+  def get_max_z
     return get_style_specific_data.max_z
-	end
+  end
 
-	def get_min_z
+  def get_min_z
     if @style_specific_data == nil
       raise 'Set your configuration data using CartoCSSHelper::Configuration.set_style_specific_data(data)'
     end
     return get_style_specific_data.min_z
-	end
+  end
 
-  def set_path_to_tilemill_project_folder(path)
+  # Link to project folder itself, not to folder containing various projects.
+  def set_path_to_cartocss_project_folder(path)
     @style_path = path
   end
 
-	def get_path_to_tilemill_project_folder
+  def get_path_to_cartocss_project_folder
     if @style_path == nil
       raise 'Set your configuration data using CartoCSSHelper::Configuration.set_style_path(path)'
     end
-		return @style_path
-	end
+    return @style_path
+  end
 
-  def get_tilemill_project_name
-    return get_path_to_tilemill_project_folder.split(File::SEPARATOR)[-1]
+  def get_cartocss_project_name
+    return get_path_to_cartocss_project_folder.split(File::SEPARATOR)[-1]
   end
 
   def get_style_file_location
-    if @style_file == nil
-      @style_file = find_style_file_location
-    end
+    @style_file = find_style_file_location if @style_file == nil
     return @style_file
   end
 
   def find_style_file_location
-    Find.find(get_path_to_tilemill_project_folder) do |path|
-      if path =~ /.*\.style$/
-        return path
-      end
+    Find.find(get_path_to_cartocss_project_folder) do |path|
+      return path if path =~ /.*\.style$/
+    end
+  end
+
+  def project_file_location
+    Find.find(get_path_to_cartocss_project_folder) do |path|
+      return path if path =~ /.*\.yaml$/
     end
   end
 
   def set_path_to_folder_for_output(path)
     @path_to_folder_for_output = path
-    puts @path_to_folder_for_output
-    puts @path_to_folder_for_cache
   end
 
   def get_path_to_folder_for_output
     if @path_to_folder_for_output == nil
       raise 'Set your configuration data using CartoCSSHelper::Configuration.set_path_to_folder_for_output(path)'
     end
-    FileUtils::mkdir_p @path_to_folder_for_output
+    FileUtils.mkdir_p @path_to_folder_for_output
     return @path_to_folder_for_output
   end
 
   def set_path_to_folder_for_cache(path)
     @path_to_folder_for_cache = path
-    puts @path_to_folder_for_output
-    puts @path_to_folder_for_cache
   end
 
   def get_path_to_folder_for_cache
     if @path_to_folder_for_cache == nil
       raise 'Set your configuration data using CartoCSSHelper::Configuration.set_path_to_folder_for_cache(path)'
     end
-    FileUtils::mkdir_p @path_to_folder_for_cache
+    FileUtils.mkdir_p @path_to_folder_for_cache
     return @path_to_folder_for_cache
   end
 
-	def get_path_to_folder_for_branch_specific_cache
-		location = File.join(get_path_to_folder_for_cache, 'generated_images', CartoCSSHelper::Git.get_commit_hash, '')
-		FileUtils::mkdir_p location
-		return location
-	end
+  def get_path_to_folder_for_branch_specific_cache
+    location = File.join(get_path_to_folder_for_cache, 'generated_images', CartoCSSHelper::Git.get_commit_hash, '')
+    FileUtils.mkdir_p location
+    return location
+  end
+
+  def get_path_to_folder_for_overpass_cache
+    location = File.join(get_path_to_folder_for_cache, 'overpass', '')
+    FileUtils.mkdir_p location
+    return location
+  end
+
+  def get_path_to_folder_for_notes_api_cache
+    location = File.join(get_path_to_folder_for_cache, 'notes-api', '')
+    FileUtils.mkdir_p location
+    return location
+  end
+
+  def get_data_filename
+    return get_path_to_folder_for_branch_specific_cache + 'data.osm'
+  end
+
+  @overpass_instance_url = 'http://overpass-api.de/api'
+  def set_overpass_instance_url(path)
+    @overpass_instance_url = path
+  end
+
+  def set_known_alternative_overpass_url
+    set_overpass_instance_url('http://overpass.osm.rambler.ru/cgi')
+  end
+
+  def get_overpass_instance_url
+    return @overpass_instance_url
+  end
+
+  def set_path_to_kosmtik(path)
+    @kosmtik_path = path
+  end
+
+  def path_to_kosmtik
+    @kosmtik_path
+  end
+
+  def default_renderer
+    :tilemill
+  end
+
+  def set_renderer(renderer)
+    @renderer = renderer
+  end
+
+  def renderer
+    return default_renderer if @renderer == nil
+    return @renderer
+  end
 
-	def get_path_to_folder_for_overpass_cache
-		location = File.join(get_path_to_folder_for_cache, 'overpass', '')
-		FileUtils::mkdir_p location
-		return location
-	end
+  def mapnik_reference_version_override
+    @mapnik_reference_version
+  end
 
-	def get_data_filename
-		return get_path_to_folder_for_branch_specific_cache+'data.osm'
-	end
-end
+  def set_mapnik_reference_version_override(version)
+    @mapnik_reference_version = version
+  end
+end

data/lib/cartocss_helper/data_file_handling.rb

@@ -1,45 +1,42 @@
 # encoding: UTF-8
 require_relative 'configuration'
+require_relative 'util/systemhelper.rb'
 
 module CartoCSSHelper
   class DataFileLoader
     @@loaded_filename = nil
     def self.get_filename_of_recently_loaded_file
-      if @@loaded_filename == Configuration.get_data_filename
-        return nil
-      end
+      return nil if @@loaded_filename == Configuration.get_data_filename
       return @@loaded_filename
     end
 
-    def self.load_data_into_database(data_filename, debug=false)
+    def self.get_command_to_load_using_osmpgsql(data_filename)
+      return "osm2pgsql --create --slim --drop --cache 10 --number-processes 1 --hstore --unlogged --style #{Configuration.get_style_file_location} --multi-geometry '#{data_filename}'"
+    end
+
+    def self.is_already_loaded(data_filename)
       if get_filename_of_recently_loaded_file == data_filename
         puts "\tavoided reloading the same file! <#{data_filename}>"
-        return
+        return true
       end
+      return false
+    end
+
+    def self.load_data_into_database(data_filename, debug = false)
+      return if is_already_loaded(data_filename)
       start_time = Time.now
       puts "\tloading data into database <#{data_filename}>"
       @@loaded_filename = nil
-      silence = '> /dev/null 2>&1'
-      if debug
-        silence = ''
-      end
-
-      command = "osm2pgsql --create --slim --cache 10 --number-processes 1 --hstore --style #{Configuration.get_style_file_location} --multi-geometry '#{data_filename}' #{silence}"
-      if debug
-        puts command
-      end
-      if !system(command)
+      begin
+        execute_command(get_command_to_load_using_osmpgsql(data_filename), debug, ignore_stderr_presence: true) # osm2pgsql outputs everything to stderr
+      rescue FailedCommandException => e
         puts 'loading data into database failed'
-        if !debug
-          puts 'retry with enabled debug'
-          load_data_into_database(data_filename, true)
-        else
-          raise 'osm2pgsql failed'
-        end
+        raise e if debug
+        puts 'retry with enabled debug'
+        load_data_into_database(data_filename, true)
       end
       @@loaded_filename = data_filename
-      time_in_seconds = Time.now - start_time
-      puts "\tloading lasted #{time_in_seconds.to_i}s"
+      puts "\tloading lasted #{(Time.now - start_time).to_i}s"
     end
   end
 
@@ -89,17 +86,17 @@ module CartoCSSHelper
     end
 
     def generate_way_topology(lat, lon, tags)
-      add_node lat, lon-@size/3, [], 1
-      add_node lat, lon+@size/3, [], 2
+      add_node lat, lon - @size / 3, [], 1
+      add_node lat, lon + @size / 3, [], 2
       add_way tags, [1, 2], 3
     end
 
     def generate_closed_way_topology(lat, lon, tags)
-      delta = @size/3
-      add_node lat-delta, lon-delta, [], 1
-      add_node lat-delta, lon+delta, [], 2
-      add_node lat+delta, lon+delta, [], 3
-      add_node lat+delta, lon-delta, [], 4
+      delta = @size / 3
+      add_node lat - delta, lon - delta, [], 1
+      add_node lat - delta, lon + delta, [], 2
+      add_node lat + delta, lon + delta, [], 3
+      add_node lat + delta, lon - delta, [], 4
       add_way tags, [1, 2, 3, 4, 1], 5
     end
 
@@ -121,19 +118,19 @@ module CartoCSSHelper
     def add_way(tags, nodes, id)
       @data_file.write "\n"
       @data_file.write "  <way id='#{id}' visible='true'>"
-      nodes.each { |node|
+      nodes.each do |node|
         @data_file.write "\n"
         @data_file.write "    <nd ref='#{node}' />"
-      }
+      end
       add_tags(tags)
       @data_file.write "\n  </way>"
     end
 
     def add_tags(tags)
-      tags.each { |tag|
+      tags.each do |tag|
         @data_file.write "\n"
         @data_file.write "    <tag k='#{tag[0]}' v='#{tag[1]}' />"
-      }
+      end
     end
   end
-end
+end