cartocss_helper 0.0.0

data/lib/cartocss_helper/visualise_changes_image_generation.rb

@@ -0,0 +1,148 @@
+require_relative 'visualise_changes_diff_from_images'
+require_relative 'git'
+require_relative 'downloader'
+
+module CartoCSSHelper
+  class VisualDiff
+    def self.visualise_changes_synthethic_test(tags, type, on_water, zlevel_range, old_branch, new_branch)
+      tags = VisualDiff.remove_magic_from_tags(tags)
+      on_water_string = ''
+      if on_water
+        on_water_string = ' on water'
+      end
+      header = "#{ VisualDiff.dict_to_pretty_tag_list(tags) } #{ type }#{ on_water_string }"
+      puts "visualise_changes_synthethic_test <#{header}> #{old_branch} -> #{new_branch}"
+      Git.checkout(old_branch)
+      old = VisualDiff.collect_images_for_synthethic_test(tags, type, on_water, zlevel_range)
+      Git.checkout(new_branch)
+      new = VisualDiff.collect_images_for_synthethic_test(tags, type, on_water, zlevel_range)
+      VisualDiff.pack_image_sets old, new, header, old_branch, new_branch, 200
+    end
+
+    def self.remove_magic_from_tags(tags)
+      magicless_tags = Hash.new
+      tags.each {|key, value|
+        if value == :any
+          value = 'any tag value'
+        end
+        magicless_tags[key]=value
+      }
+      return magicless_tags
+    end
+
+    def self.collect_images_for_synthethic_test(tags, type, on_water, zlevel_range)
+      collection = []
+      zlevel_range.each { |zlevel|
+        scene = Scene.new(tags, zlevel, on_water, type)
+        collection.push(ImageForComparison.new(scene.get_image_filename, "z#{zlevel}"))
+      }
+      return collection
+    end
+
+    class RealDataSource
+      def initialize(latitude, longitude, wanted_download_bbox_size)
+        wanted_download_bbox_size = [wanted_download_bbox_size, 0.001].max
+        wanted_download_bbox_size = [wanted_download_bbox_size, 0.400].min #0.5 is too big for Utrecht
+        @download_bbox_size = wanted_download_bbox_size
+        @latitude = latitude
+        @longitude = longitude
+        @data_filename = Downloader.download_osm_data_for_location(@latitude, @longitude, @download_bbox_size)
+        @loaded = false
+      end
+
+      def load
+        if !@loaded
+          puts "\tloading data into database"
+          DataFileLoader.load_data_into_database(@data_filename)
+          puts "\tgenerating images"
+          @loaded = true
+        end
+      end
+
+      def get_timestamp
+        f = File.new(@data_filename)
+        return f.mtime.to_i
+      end
+    end
+
+    def self.visualise_changes_on_real_data(tags, type, wanted_latitude, wanted_longitude, zlevels, old_branch, new_branch)
+      #special support for following tag values:  :any_value
+      header = "#{ VisualDiff.dict_to_pretty_tag_list(tags) } #{type} #{ wanted_latitude } #{ wanted_longitude } #{zlevels}"
+      puts "visualise_changes_on_real_data <#{header}> #{old_branch} -> #{new_branch}"
+      latitude, longitude = Downloader.locate_element_with_given_tags_and_type tags, type, wanted_latitude, wanted_longitude
+      download_bbox_size = VisualDiff.scale_inverse zlevels.min, 0.03, 14
+      source = RealDataSource.new(latitude, longitude, download_bbox_size)
+      Git.checkout old_branch
+      old = VisualDiff.collect_images_for_real_data_test(tags, type, latitude, longitude, zlevels, source)
+      Git.checkout new_branch
+      new = VisualDiff.collect_images_for_real_data_test(tags, type, latitude, longitude, zlevels, source)
+      header = "#{ VisualDiff.dict_to_pretty_tag_list(tags) } #{type} #{ wanted_latitude } #{ wanted_longitude } #{zlevels}"
+      VisualDiff.pack_image_sets old, new, header, old_branch, new_branch, 400
+    end
+
+    def self.scale_inverse(zlevel, reference_value, reference_zlevel)
+      return VisualDiff.scale reference_zlevel, reference_value, zlevel
+    end
+
+    def self.scale(zlevel, reference_value, reference_zlevel)
+      diff = zlevel - reference_zlevel
+      rescaler = 2.0**diff
+      return reference_value*rescaler
+    end
+
+    def self.collect_images_for_real_data_test(tags, type, latitude, longitude, zlevels, source)
+      collection = []
+      zlevels.each { |zlevel|
+        image_size_for_16_zoom_level = 1000
+        image_size = (VisualDiff.scale zlevel, image_size_for_16_zoom_level, 16)
+        mutiplier = 1000
+        image_size = (image_size*mutiplier).to_int
+        render_bbox_size = 0.015
+        wanted_image_size = 400
+        ratio = 1.0*image_size/(wanted_image_size*mutiplier)
+        image_size /= ratio
+        render_bbox_size /= ratio
+        image_size /= mutiplier
+        image_size = image_size.to_i
+        if image_size!=wanted_image_size
+          puts VisualDiff.scale zlevel, image_size_for_16_zoom_level, 16
+          puts zlevel
+          puts image_size
+          puts wanted_image_size
+          puts ratio
+          raise "#{image_size} mismatches #{wanted_image_size}"
+        end
+        filename = "#{CartoCSSHelper::Configuration.get_path_to_folder_for_branch_specific_cache+"#{tags.to_a.to_s} #{type} #{latitude} #{longitude} #{zlevel}zlevel #{image_size}px #{source.get_timestamp}.png"}"
+        if !File.exists?(filename)
+          source.load
+          Scene.run_tilemill_export_image(latitude, longitude, zlevel, render_bbox_size, image_size, filename)
+        end
+        collection.push(ImageForComparison.new(filename, "z#{zlevel}"))
+      }
+      return collection
+    end
+
+    def self.pack_image_sets(old, new, header, old_branch, new_branch, image_size)
+      old_branch = old_branch.gsub(/[\x00\/\\:\*\?\"<>\|]/, '_')
+      new_branch = new_branch.gsub(/[\x00\/\\:\*\?\"<>\|]/, '_')
+      filename_sufix = "#{ old_branch } -> #{ new_branch }"
+      filename = CartoCSSHelper::Configuration.get_path_to_folder_for_output + "#{header} #{filename_sufix}.png"
+      diff = FullSetOfComparedImages.new(old, new, header, filename, image_size)
+      diff.save
+    end
+    def self.dict_to_pretty_tag_list(dict)
+      result = ''
+      dict.to_a.each { |tag|
+        if result != ''
+          result << '; '
+        end
+        value = "'#{tag[1]}'"
+        if tag[1] == :any_value
+          value = '{any value}'
+        end
+        result << "#{tag[0]}=#{value}"
+      }
+      return result
+    end
+  end
+end

data/lib/cartocss_helper.rb

@@ -0,0 +1,70 @@
+# encoding: UTF-8
+require 'open3'
+
+module CartoCSSHelper
+end
+require_relative 'cartocss_helper/tag_lister.rb'
+require_relative 'cartocss_helper/visualise_changes_image_generation.rb'
+require_relative 'cartocss_helper/downloader.rb'
+require_relative 'cartocss_helper/data_file_handling.rb'
+require_relative 'cartocss_helper/validator.rb'
+require_relative 'cartocss_helper/git.rb'
+include CartoCSSHelper::Validator
+include CartoCSSHelper::Git
+
+module CartoCSSHelper
+  def  self.test_tag_on_real_data(tags, zlevels, old_branch, new_branch)
+    ['node', 'closed_way', 'way'].each {|type|
+      test_tag_on_real_data_for_this_type(tags, zlevels, old_branch, new_branch, type)
+    }
+  end
+
+  def self.test_tag_on_real_data_for_this_type(tags, zlevels, old_branch, new_branch, type)
+    #special support for following tag values:  :any_value
+    krakow_latitude = 50.1
+    krakow_longitude = 19.9
+    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, krakow_latitude, krakow_longitude, zlevels, old_branch, new_branch)
+    japan_latitude = 36.1
+    japan_longitude = 140.7
+    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, japan_latitude, japan_longitude, zlevels, old_branch, new_branch)
+    russia_latitude = 54.8
+    russia_longitude = 31.7
+    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, russia_latitude, russia_longitude, zlevels, old_branch, new_branch)
+    mecca_latitude = 21.3
+    mecca_longitude = 39.5
+    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, mecca_latitude, mecca_longitude, zlevels, old_branch, new_branch)
+    georgia_latitude = 41.4
+    georgia_longitude = 44.5
+    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, georgia_latitude, georgia_longitude, zlevels, old_branch, new_branch)
+    london_latitude = 51.5
+    london_longitude = -0.1
+    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, london_latitude, london_longitude, zlevels, old_branch, new_branch)
+    #TODO: solve problems with respons to big to store in memory
+    #utrecht_latitude =  52.09
+    #utrecht_longitude = 5.11
+    #CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, utrecht_latitude, utrecht_longitude, zlevels, old_branch, new_branch)
+    rural_uk_latitude =  53.2
+    rural_uk_longitude = -1.8
+    CartoCSSHelper::VisualDiff.visualise_changes_on_real_data(tags, type, rural_uk_latitude, rural_uk_longitude, zlevels, old_branch, new_branch)
+  end
+
+  def self.add_common_secondary_tags(tags)
+    return tags.merge({'name' => 'ÉÉÉÉÉÉ ÉÉÉÉÉÉ ÉÉÉÉÉÉ', 'ref' => '1', 'ele' => '8000', 'operator' => 'operator ÉÉ ÉÉ ÉÉ', 'brand' => 'brand ÉÉ ÉÉ ÉÉ ÉÉ'})
+  end
+
+  def self.test(tags, new_branch, old_brach='master', test_on_water=false, zlevels=Configuration.get_min_z..Configuration.get_max_z)
+    puts "processing #{VisualDiff.dict_to_pretty_tag_list(tags)}"
+    syn_tags = add_common_secondary_tags(tags)
+    ['node', 'closed_way', 'way'].each {|type|
+      CartoCSSHelper::VisualDiff.visualise_changes_synthethic_test(syn_tags, type, test_on_water, zlevels, old_brach, new_branch)
+    }
+    test_tag_on_real_data(tags, zlevels, old_brach, new_branch)
+  end
+
+  def self.probe(tags, new_branch, old_brach='master',test_on_water=false, zlevels=Configuration.get_min_z..Configuration.get_max_z)
+    syn_tags = add_common_secondary_tags(tags)
+    ['node', 'closed_way', 'way'].each {|type|
+      CartoCSSHelper::VisualDiff.visualise_changes_synthethic_test(syn_tags, type, test_on_water, zlevels, old_brach, new_branch)
+    }
+  end
+end

data/license.txt

@@ -0,0 +1,9 @@
+This software and associated documentation files (the "Software") is
+released under the CC0 Public Domain Dedication, version 1.0, as
+published by Creative Commons. To the extent possible under law, the
+author(s) have dedicated all copyright and related and neighboring
+rights to the Software to the public domain worldwide. The Software is
+distributed WITHOUT ANY WARRANTY.
+If you did not receive a copy of the CC0 Public Domain Dedication
+along with the Software, see
+<http://creativecommons.org/publicdomain/zero/1.0/>

data/readme.md

@@ -0,0 +1,25 @@
+##CartoCSS Helper
+
+It is a tool making development of CartoCSS styles more efficient. Automates actions necessary to produce test images and validates style. Loading data using osm2pgsql, rendering with TileMill, obtaining test data from overpass turbo.
+
+Proper testing of map rendering requires testing covering mutiple situations. Doing it manually means preparing test data and locating places with applicable real data, loading map data into database, then moving viewport of TileMill to check every data location across multiple zoom levels. It is painfully slow.
+
+Fortunately it is possible to automate it almost entirely. With this tool testing new rendering change requires only specification of tag combination that should be tested and type of element.
+
+It is work in progress, major problems that should be solved include:
+
+* Lack of documentation
+* Executing git checkout commands on map style git repository during normal script operation (https://github.com/matkoniecz/CartoCSSHelper/issues/9)
+* Current rendering method is obnoxiously slow. (https://github.com/matkoniecz/CartoCSSHelper/issues/1)
+
+###Example
+
+`require cartocss_helper`
+`tags = {'landuse' => 'village_green', 'tourism' => 'attraction'}`
+`CartoCSSHelper.test tags, 'master', 'v2.28.1'`
+
+Runs quick test for specified tag combination rendering only this element, followed by locating multiple places across globe where such tags are used. For each test case images are produced both for current `master` branch and release `v2.28.1` across multiple zoom levels. Finally tool generates before/after comparisons for each case. Some of generated images were used in https://github.com/gravitystorm/openstreetmap-carto/issues/1371.
+
+###Automated stuff
+
+[![Code Climate](https://codeclimate.com/github/mkoniecz/CartoCSSHelper/badges/gpa.svg)](https://codeclimate.com/github/mkoniecz/CartoCSSHelper)

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: cartocss_helper
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Mateusz Konieczny
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.3
+- !ruby/object:Gem::Dependency
+  name: sys-filesystem
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.4
+- !ruby/object:Gem::Dependency
+  name: rmagick
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.13.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.13.4
+description: Tool to make development of CartoCSS styles more efficient. Automates
+  actions necessary to produce test images and validates style. Loading data using
+  osm2pgsql, rendering with TileMill, obtaining test data from overpass turbo.
+email:
+- matkoniecz@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/cartocss_helper.rb
+- lib/cartocss_helper/visualise_changes_image_generation.rb
+- lib/cartocss_helper/tag_lister.rb
+- lib/cartocss_helper/image_generator.rb
+- lib/cartocss_helper/validator.rb
+- lib/cartocss_helper/downloader.rb
+- lib/cartocss_helper/git.rb
+- lib/cartocss_helper/heuristic.rb
+- lib/cartocss_helper/visualise_changes_diff_from_images.rb
+- lib/cartocss_helper/configuration.rb
+- lib/cartocss_helper/style_specific/style_specific.rb
+- lib/cartocss_helper/style_specific/default_osm_style.rb
+- lib/cartocss_helper/data_file_handling.rb
+- license.txt
+- readme.md
+homepage: https://github.com/matkoniecz/CartoCSSHelper
+licenses:
+- CC0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.8.23
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Tool to make development of CartoCSS styles more efficient.
+test_files: []