cartocss_helper 0.0.0

data/lib/cartocss_helper/configuration.rb

@@ -0,0 +1,103 @@
+# encoding: UTF-8
+require 'fileutils'
+require 'find'
+
+require_relative 'style_specific/default_osm_style.rb'
+
+module CartoCSSHelper::Configuration
+  def set_style_specific_data(data)
+    @style_specific_data = data
+  end
+
+  def get_style_specific_data
+    if @style_specific_data == nil
+      raise 'Set your configuration data using CartoCSSHelper::Configuration.set_style_specific_data(data)'
+    end
+    return @style_specific_data
+  end
+
+	def get_max_z
+    return get_style_specific_data.max_z
+	end
+
+	def get_min_z
+    if @style_specific_data == nil
+      raise 'Set your configuration data using CartoCSSHelper::Configuration.set_style_specific_data(data)'
+    end
+    return get_style_specific_data.min_z
+	end
+
+  def set_path_to_tilemill_project_folder(path)
+    @style_path = path
+  end
+
+	def get_path_to_tilemill_project_folder
+    if @style_path == nil
+      raise 'Set your configuration data using CartoCSSHelper::Configuration.set_style_path(path)'
+    end
+		return @style_path
+	end
+
+  def get_tilemill_project_name
+    return get_path_to_tilemill_project_folder.split(File::SEPARATOR)[-1]
+  end
+
+  def get_style_file_location
+    if @style_file == nil
+      @style_file = find_style_file_location
+    end
+    return @style_file
+  end
+
+  def find_style_file_location
+    Find.find(get_path_to_tilemill_project_folder) do |path|
+      if path =~ /.*\.style$/
+        return path
+      end
+    end
+  end
+
+  def set_path_to_folder_for_output(path)
+    @path_to_folder_for_output = path
+    puts @path_to_folder_for_output
+    puts @path_to_folder_for_cache
+  end
+
+  def get_path_to_folder_for_output
+    if @path_to_folder_for_output == nil
+      raise 'Set your configuration data using CartoCSSHelper::Configuration.set_path_to_folder_for_output(path)'
+    end
+    FileUtils::mkdir_p @path_to_folder_for_output
+    return @path_to_folder_for_output
+  end
+
+  def set_path_to_folder_for_cache(path)
+    @path_to_folder_for_cache = path
+    puts @path_to_folder_for_output
+    puts @path_to_folder_for_cache
+  end
+
+  def get_path_to_folder_for_cache
+    if @path_to_folder_for_cache == nil
+      raise 'Set your configuration data using CartoCSSHelper::Configuration.set_path_to_folder_for_cache(path)'
+    end
+    FileUtils::mkdir_p @path_to_folder_for_cache
+    return @path_to_folder_for_cache
+  end
+
+	def get_path_to_folder_for_branch_specific_cache
+		location = File.join(get_path_to_folder_for_cache, CartoCSSHelper::Git.get_commit_hash, '')
+		FileUtils::mkdir_p location
+		return location
+	end
+
+	def get_path_to_folder_for_overpass_cache
+		location = File.join(get_path_to_folder_for_cache, 'overpass', '')
+		FileUtils::mkdir_p location
+		return location
+	end
+
+	def get_data_filename
+		return get_path_to_folder_for_branch_specific_cache+'data.osm'
+	end
+end

data/lib/cartocss_helper/data_file_handling.rb

@@ -0,0 +1,97 @@
+# encoding: UTF-8
+require_relative 'configuration'
+
+module CartoCSSHelper
+  class DataFileLoader
+    def self.load_data_into_database(data_filename, debug=false)
+      silence = '> /dev/null 2>&1'
+      if debug
+        silence = ''
+      end
+
+      command = "osm2pgsql --create --slim --cache 10 --number-processes 1 --hstore --style #{Configuration.get_style_file_location} --multi-geometry '#{data_filename}' #{silence}"
+      if debug
+        puts command
+      end
+      system command
+    end
+  end
+
+  class DataFileGenerator
+    def initialize(tags, type, lat, lon, size)
+      @lat = lat
+      @lon = lon
+      @tags = tags
+      @type = type
+      @size = size
+    end
+
+    def generate
+      @data_file = open(Configuration.get_data_filename, 'w')
+      generate_prefix
+      if @type == 'node'
+        generate_node_topology(@lat, @lon, @tags)
+      elsif @type == 'way'
+        generate_way_topology(@lat, @lon, @tags)
+      elsif @type == 'closed_way'
+        generate_closed_way_topology(@lat, @lon, @tags)
+      else
+        raise 'this type of element does not exists'
+      end
+      generate_sufix
+      @data_file.close
+    end
+
+    def generate_node_topology(lat, lon, tags)
+      add_node lat, lon, tags, 2387
+    end
+
+    def generate_way_topology(lat, lon, tags)
+      add_node lat, lon-@size/3, [], 1
+      add_node lat, lon+@size/3, [], 2
+      add_way tags, [1, 2], 3
+    end
+
+    def generate_closed_way_topology(lat, lon, tags)
+      delta = @size/3
+      add_node lat-delta, lon-delta, [], 1
+      add_node lat-delta, lon+delta, [], 2
+      add_node lat+delta, lon+delta, [], 3
+      add_node lat+delta, lon-delta, [], 4
+      add_way tags, [1, 2, 3, 4, 1], 5
+    end
+
+    def generate_prefix
+      @data_file.write "<?xml version='1.0' encoding='UTF-8'?>\n<osm version='0.6' generator='script'>"
+    end
+
+    def generate_sufix
+      @data_file.write "\n</osm>"
+    end
+
+    def add_node(lat, lon, tags, id)
+      @data_file.write "\n"
+      @data_file.write "  <node id='#{id}' visible='true' lat='#{lat}' lon='#{lon}'>"
+      add_tags(tags)
+      @data_file.write '</node>'
+    end
+
+    def add_way(tags, nodes, id)
+      @data_file.write "\n"
+      @data_file.write "  <way id='#{id}' visible='true'>"
+      nodes.each { |node|
+        @data_file.write "\n"
+        @data_file.write "    <nd ref='#{node}' />"
+      }
+      add_tags(tags)
+      @data_file.write "\n  </way>"
+    end
+
+    def add_tags(tags)
+      tags.each { |tag|
+        @data_file.write "\n"
+        @data_file.write "    <tag k='#{tag[0]}' v='#{tag[1]}' />"
+      }
+    end
+  end
+end

data/lib/cartocss_helper/downloader.rb

@@ -0,0 +1,198 @@
+# encoding: UTF-8
+require 'rest-client'
+require 'digest/sha1'
+require 'sys/filesystem'
+
+
+module CartoCSSHelper
+  class Downloader
+    def self.download_osm_data_for_location(latitude, longitude, size, accept_cache=true)
+      filename = CartoCSSHelper::Configuration.get_path_to_folder_for_cache + "#{latitude} #{longitude} #{size}.osm"
+      if File.exists?(filename)
+        if accept_cache
+          return filename
+        end
+        File.delete(filename)
+      end
+      query = get_query_to_download_data_around_location(latitude, longitude, size)
+      text = get_overpass_query_results(query)
+      file = File.new(filename, 'w')
+      file.write text
+      file.close
+      return filename
+    end
+
+    def self.get_query_to_download_data_around_location(latitude, longitude, size)
+      min_latitude = latitude - size/2
+      max_latitude = latitude + size/2
+      min_longitude = longitude - size/2
+      max_longitude = longitude + size/2
+      bb = "#{min_latitude},#{min_longitude},#{max_latitude},#{max_longitude}"
+      query = '[timeout:3600];'
+      query += "\n"
+      query += "(node(#{bb});<;);"
+      query += "\n"
+      query += 'out meta;'
+      query += "\n"
+      query += '/*'
+      query += "\nbbox size: #{size}"
+      query += "\nhttp://www.openstreetmap.org/#map=17/#{latitude}/#{longitude}"
+      query += "\n"
+      query += '*/'
+      query += "\n"
+      return query
+    end
+
+    def self.locate_element_with_given_tags_and_type(tags, type, latitude, longitude)
+      max_range_in_km_for_radius = 100 #"Radius temporarly limited to 100 km to mitigate a bug.". Previously Overpass crashed for values larger than 3660
+
+      #special support for following tag values:  :any_value
+      range = 10*1000
+      loop do
+        list = Downloader.get_overpass_query_results(Downloader.get_query_to_get_location(tags, type, latitude, longitude, range))
+        if list.length != 0
+          return self.list_returned_by_overpass_to_a_single_location(list)
+        end
+        range=range+[range, 100000].min
+        if range >= max_range_in_km_for_radius*1000
+          list = Downloader.get_overpass_query_results(Downloader.get_query_to_get_location(tags, type, latitude, longitude, :infinity))
+          if list.length != 0
+            return self.list_returned_by_overpass_to_a_single_location(list)
+          else
+            raise 'failed to find such location'
+          end
+        end
+      end
+    end
+
+    def self.list_returned_by_overpass_to_a_single_location(list)
+      list = list.match(/((|-)[\d\.]+)\s+((|-)[\d\.]+)/).to_a
+      lat = Float(list[1])
+      lon = Float(list[3])
+      return lat, lon
+    end
+
+    def self.get_query_to_get_location(tags, type, latitude, longitude, range)
+      #special support for following tag values:  :any_value
+      locator = '[timeout:3600][out:csv(::lat,::lon;false)];'
+      locator += "\n"
+      if type == 'closed_way'
+        type = 'way'
+      end
+      locator += Downloader.get_query_element_to_get_location(tags, latitude, longitude, type, range)
+      locator +='out center;'
+      locator += "\n"
+      locator += '/*'
+      range_string = ''
+      if range == :infinity
+        range_string = 'infinity'
+      else
+        range_string = "#{range/1000}km"
+      end
+      locator += "\nrange: #{range_string}"
+      locator += "\nhttp://www.openstreetmap.org/#map=17/#{latitude}/#{longitude}"
+      locator += "\n"
+      locator += '*/'
+      locator += "\n"
+      return locator
+    end
+
+    def self.get_query_element_to_get_location(tags, latitude, longitude, type, range)
+      #special support for following tag values:  :any_value
+      #TODO - escape value with quotation signs in them
+      element="(#{type}"
+      element += "\n"
+      tags.each {|tag|
+        if tag[1] == :any_value
+          element+="\t['#{tag[0]}']"
+        else
+          element+="\t['#{tag[0]}'='#{tag[1]}']"
+        end
+        element += "\n"
+      }
+      if range != :infinity
+        element+="\t(around:#{range},#{latitude},#{longitude});"
+        element += "\n"
+      end
+      element+=');'
+      element += "\n"
+      return element
+    end
+
+    def self.get_overpass_query_results(query, debug=false)
+      cached = get_overpass_query_results_from_cache(query)
+      return cached unless cached == nil
+
+      check_for_free_space
+
+      puts 'Running Overpass query'
+      if debug
+        puts query
+        puts
+      end
+      cached = Downloader.run_overpass_query query
+      file = File.new(get_query_cache_filename(query), 'w')
+      file.write cached
+      file.close
+      return cached
+    end
+
+    def self.get_overpass_query_results_from_cache(query)
+      query_cache_filename = get_query_cache_filename(query)
+      if File.exists?(query_cache_filename)
+        file = File.new(query_cache_filename)
+        cached = file.read
+        file.close
+        return cached
+      end
+      return nil
+    end
+
+    def self.get_query_cache_filename(query)
+      # noinspection RubyResolve
+      hash = Digest::SHA1.hexdigest query
+      query_cache_filename = CartoCSSHelper::Configuration.get_path_to_folder_for_overpass_cache + hash + '_query.cache'
+      return query_cache_filename
+    end
+
+    def self.check_for_free_space
+      stat = Sys::Filesystem.stat(CartoCSSHelper::Configuration.get_path_to_folder_for_overpass_cache)
+      gb_available = stat.block_size * stat.blocks_available / 1024 / 1024 / 1024
+      if gb_available < 2
+        #TODO cleanup cache rather than crash
+        raise 'less than 2GB of free space on disk with cache folder'
+      end
+    end
+
+    def self.run_overpass_query(query, retry_count=0, retry_max=5)
+      #default timeout is set to 180: http://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
+      #[timeout:3600]; should be used in all queries executed here
+      start = Time.now.to_s
+      begin
+        return RestClient::Request.execute(:method => :get, :url => Downloader.format_query_into_url(query), :timeout => 3600)
+      rescue RestClient::RequestFailed => e
+        puts query
+        puts e.response
+        puts e.http_code
+        puts start
+        puts Time.now.to_s
+        if retry_count < retry_max
+          sleep 60*5
+          Downloader.run_overpass_query(query, retry_count+1, retry_max)
+        else
+          e.raise
+        end
+      rescue ArgumentError => e
+        puts 'ArgumentError from rest-client, most likely caused by https://github.com/rest-client/rest-client/issues/359'
+        puts 'try overpass query that will return smaller amount of data'
+        e.raise
+      end
+    end
+
+    def self.format_query_into_url(query)
+      query = query.gsub(/\n/, '')
+      query = query.gsub(/\t/, '')
+      return 'http://overpass-api.de/api/interpreter?data=' + URI.escape(query)
+    end
+  end
+end

data/lib/cartocss_helper/git.rb

@@ -0,0 +1,32 @@
+require_relative 'configuration.rb'
+
+module CartoCSSHelper
+  module Git
+    def checkout(branch, debug=false)
+      silence = '> /dev/null 2>&1'
+      if debug
+        silence = ''
+      end
+      Dir.chdir(Configuration.get_path_to_tilemill_project_folder) {
+        require 'open3'
+        if !system("git checkout #{branch} #{silence}")
+          raise 'failed checkout'
+        end
+      }
+    end
+
+    def get_commit_hash
+      Dir.chdir(Configuration.get_path_to_tilemill_project_folder) {
+        Open3.popen3('git log -n 1 --pretty=format:"%H"') {|_, stdout, stderr, _|
+          commit = stdout.read.chomp
+          error = stderr.read.chomp
+          if error != ''
+            raise error
+          end
+          return commit
+        }
+      }
+      raise 'impossible happened'
+    end
+  end
+end

data/lib/cartocss_helper/heuristic.rb

@@ -0,0 +1,117 @@
+module CartoCSSHelper
+  module Heuristic
+    require 'set'
+    require_relative 'configuration.rb'
+    include CartoCSSHelper::Configuration
+    def get_tags
+      tags = get_tags_from_mss
+      tags.merge(get_tags_from_yaml)
+      tags.merge(get_tags_from_osm2pqsql)
+      #unrecovable = Set.new [['area', 'yes'], ['area', 'no']] #TODO - how this should be handled?
+      #tags.merge(unrecovable)
+      return tags.to_a.sort
+    end
+
+    def get_tags_from_mss
+      tags = Set.new
+      filenames = Dir[Configuration.get_path_to_tilemill_project_folder+'*.mss']
+      filenames.each { |filename|
+        tags.merge(get_tags_from_mss_file filename)
+      }
+      return tags
+    end
+
+    def get_tags_from_yaml
+      tags = Set.new
+      filenames = Dir[Configuration.get_path_to_tilemill_project_folder+'*.yaml']
+      filenames.each { |filename|
+        tags.merge(get_tags_from_yaml_file filename)
+      }
+      return  tags
+    end
+
+    def get_tags_from_osm2pqsql
+      tags = Set.new
+      filenames = Dir[Configuration.get_path_to_tilemill_project_folder+'*.style']
+      filenames.each { |filename|
+        tags.merge(get_tags_from_osm2pqsql_file filename)
+      }
+      return  tags
+    end
+
+    def get_tags_from_mss_file(style_filename)
+      possible_key_values = get_tags_from_osm2pqsql
+      tags = Set.new
+      #puts style_filename
+      style_file = open(style_filename)
+      style = style_file.read
+      matched = style.scan(/\[feature = '(man_made|[^_]+)_([^']+)'\]/)
+      matched.each { |element|
+        key = element[0]
+        if possible_key_values.include?([key, get_generic_tag_value])
+          tags.add([key, element[1]])
+        end
+      }
+      matched = style.scan(/(\w+) = '(\w+)'/)
+      matched.each { |element|
+        key = element[0]
+        if key != 'feature'
+          if possible_key_values.include?([key, get_generic_tag_value])
+            tags.add([key, element[1]])
+          end
+        end
+      }
+      return tags
+    end
+
+    def get_tags_from_yaml_file(yaml_filename)
+      possible_key_values = get_tags_from_osm2pqsql
+      tags = Set.new
+      yaml_file = open(yaml_filename)
+      yaml = yaml_file.read
+      #(.*\.|)     #WHERE p.highway = 'turning_circle'
+      #("|)        #natural and other SQL keywords
+      #([^"() ]+)  #tag
+      #repeat
+      # =
+      #'([^'()]+)' #quoted value
+      matched = yaml.scan(/(.*\.|)("|)([^"() ]+)("|) = '([^'()]+)'/)
+      matched.each { |element|
+        key = element[2]
+        value = element[4]
+        if possible_key_values.include?([key, get_generic_tag_value])
+          tags.add([key, value])
+        end
+      }
+      matched = yaml.scan(/("|)([^"() ]+)("|) (NOT |)IN \(([^()]*)\)/)
+      matched.each { |element|
+        tag = element[1]
+        key = tag.gsub(/.*\./, '')
+        values = element[4]
+        values_matched = values.scan(/'([^']+)'/)
+        if possible_key_values.include?([key, get_generic_tag_value])
+          values_matched.each { |value|
+            tags.add([key, value[0]])
+          }
+        end
+      }
+      return tags
+    end
+
+    def get_generic_tag_value
+      return '*'
+    end
+
+    def get_tags_from_osm2pqsql_file(style_filename)
+      tags = Set.new
+      #puts style_filename
+      style_file = open(style_filename)
+      style = style_file.read
+      matched = style.scan(/^(node,way|node|way)\s*([^ ]+)\s*text\s*($|linear|polygon|nocache)/)
+      matched.each { |element|
+        tags.add([element[1], get_generic_tag_value])
+      }
+      return tags
+    end
+  end
+end