cartocss_helper 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,103 @@
1
+ # encoding: UTF-8
2
+ require 'fileutils'
3
+ require 'find'
4
+
5
+ require_relative 'style_specific/default_osm_style.rb'
6
+
7
+ module CartoCSSHelper::Configuration
8
+ def set_style_specific_data(data)
9
+ @style_specific_data = data
10
+ end
11
+
12
+ def get_style_specific_data
13
+ if @style_specific_data == nil
14
+ raise 'Set your configuration data using CartoCSSHelper::Configuration.set_style_specific_data(data)'
15
+ end
16
+ return @style_specific_data
17
+ end
18
+
19
+ def get_max_z
20
+ return get_style_specific_data.max_z
21
+ end
22
+
23
+ def get_min_z
24
+ if @style_specific_data == nil
25
+ raise 'Set your configuration data using CartoCSSHelper::Configuration.set_style_specific_data(data)'
26
+ end
27
+ return get_style_specific_data.min_z
28
+ end
29
+
30
+ def set_path_to_tilemill_project_folder(path)
31
+ @style_path = path
32
+ end
33
+
34
+ def get_path_to_tilemill_project_folder
35
+ if @style_path == nil
36
+ raise 'Set your configuration data using CartoCSSHelper::Configuration.set_style_path(path)'
37
+ end
38
+ return @style_path
39
+ end
40
+
41
+ def get_tilemill_project_name
42
+ return get_path_to_tilemill_project_folder.split(File::SEPARATOR)[-1]
43
+ end
44
+
45
+ def get_style_file_location
46
+ if @style_file == nil
47
+ @style_file = find_style_file_location
48
+ end
49
+ return @style_file
50
+ end
51
+
52
+ def find_style_file_location
53
+ Find.find(get_path_to_tilemill_project_folder) do |path|
54
+ if path =~ /.*\.style$/
55
+ return path
56
+ end
57
+ end
58
+ end
59
+
60
+ def set_path_to_folder_for_output(path)
61
+ @path_to_folder_for_output = path
62
+ puts @path_to_folder_for_output
63
+ puts @path_to_folder_for_cache
64
+ end
65
+
66
+ def get_path_to_folder_for_output
67
+ if @path_to_folder_for_output == nil
68
+ raise 'Set your configuration data using CartoCSSHelper::Configuration.set_path_to_folder_for_output(path)'
69
+ end
70
+ FileUtils::mkdir_p @path_to_folder_for_output
71
+ return @path_to_folder_for_output
72
+ end
73
+
74
+ def set_path_to_folder_for_cache(path)
75
+ @path_to_folder_for_cache = path
76
+ puts @path_to_folder_for_output
77
+ puts @path_to_folder_for_cache
78
+ end
79
+
80
+ def get_path_to_folder_for_cache
81
+ if @path_to_folder_for_cache == nil
82
+ raise 'Set your configuration data using CartoCSSHelper::Configuration.set_path_to_folder_for_cache(path)'
83
+ end
84
+ FileUtils::mkdir_p @path_to_folder_for_cache
85
+ return @path_to_folder_for_cache
86
+ end
87
+
88
+ def get_path_to_folder_for_branch_specific_cache
89
+ location = File.join(get_path_to_folder_for_cache, CartoCSSHelper::Git.get_commit_hash, '')
90
+ FileUtils::mkdir_p location
91
+ return location
92
+ end
93
+
94
+ def get_path_to_folder_for_overpass_cache
95
+ location = File.join(get_path_to_folder_for_cache, 'overpass', '')
96
+ FileUtils::mkdir_p location
97
+ return location
98
+ end
99
+
100
+ def get_data_filename
101
+ return get_path_to_folder_for_branch_specific_cache+'data.osm'
102
+ end
103
+ end
@@ -0,0 +1,97 @@
1
+ # encoding: UTF-8
2
+ require_relative 'configuration'
3
+
4
+ module CartoCSSHelper
5
+ class DataFileLoader
6
+ def self.load_data_into_database(data_filename, debug=false)
7
+ silence = '> /dev/null 2>&1'
8
+ if debug
9
+ silence = ''
10
+ end
11
+
12
+ command = "osm2pgsql --create --slim --cache 10 --number-processes 1 --hstore --style #{Configuration.get_style_file_location} --multi-geometry '#{data_filename}' #{silence}"
13
+ if debug
14
+ puts command
15
+ end
16
+ system command
17
+ end
18
+ end
19
+
20
+ class DataFileGenerator
21
+ def initialize(tags, type, lat, lon, size)
22
+ @lat = lat
23
+ @lon = lon
24
+ @tags = tags
25
+ @type = type
26
+ @size = size
27
+ end
28
+
29
+ def generate
30
+ @data_file = open(Configuration.get_data_filename, 'w')
31
+ generate_prefix
32
+ if @type == 'node'
33
+ generate_node_topology(@lat, @lon, @tags)
34
+ elsif @type == 'way'
35
+ generate_way_topology(@lat, @lon, @tags)
36
+ elsif @type == 'closed_way'
37
+ generate_closed_way_topology(@lat, @lon, @tags)
38
+ else
39
+ raise 'this type of element does not exists'
40
+ end
41
+ generate_sufix
42
+ @data_file.close
43
+ end
44
+
45
+ def generate_node_topology(lat, lon, tags)
46
+ add_node lat, lon, tags, 2387
47
+ end
48
+
49
+ def generate_way_topology(lat, lon, tags)
50
+ add_node lat, lon-@size/3, [], 1
51
+ add_node lat, lon+@size/3, [], 2
52
+ add_way tags, [1, 2], 3
53
+ end
54
+
55
+ def generate_closed_way_topology(lat, lon, tags)
56
+ delta = @size/3
57
+ add_node lat-delta, lon-delta, [], 1
58
+ add_node lat-delta, lon+delta, [], 2
59
+ add_node lat+delta, lon+delta, [], 3
60
+ add_node lat+delta, lon-delta, [], 4
61
+ add_way tags, [1, 2, 3, 4, 1], 5
62
+ end
63
+
64
+ def generate_prefix
65
+ @data_file.write "<?xml version='1.0' encoding='UTF-8'?>\n<osm version='0.6' generator='script'>"
66
+ end
67
+
68
+ def generate_sufix
69
+ @data_file.write "\n</osm>"
70
+ end
71
+
72
+ def add_node(lat, lon, tags, id)
73
+ @data_file.write "\n"
74
+ @data_file.write " <node id='#{id}' visible='true' lat='#{lat}' lon='#{lon}'>"
75
+ add_tags(tags)
76
+ @data_file.write '</node>'
77
+ end
78
+
79
+ def add_way(tags, nodes, id)
80
+ @data_file.write "\n"
81
+ @data_file.write " <way id='#{id}' visible='true'>"
82
+ nodes.each { |node|
83
+ @data_file.write "\n"
84
+ @data_file.write " <nd ref='#{node}' />"
85
+ }
86
+ add_tags(tags)
87
+ @data_file.write "\n </way>"
88
+ end
89
+
90
+ def add_tags(tags)
91
+ tags.each { |tag|
92
+ @data_file.write "\n"
93
+ @data_file.write " <tag k='#{tag[0]}' v='#{tag[1]}' />"
94
+ }
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,198 @@
1
+ # encoding: UTF-8
2
+ require 'rest-client'
3
+ require 'digest/sha1'
4
+ require 'sys/filesystem'
5
+
6
+
7
+ module CartoCSSHelper
8
+ class Downloader
9
+ def self.download_osm_data_for_location(latitude, longitude, size, accept_cache=true)
10
+ filename = CartoCSSHelper::Configuration.get_path_to_folder_for_cache + "#{latitude} #{longitude} #{size}.osm"
11
+ if File.exists?(filename)
12
+ if accept_cache
13
+ return filename
14
+ end
15
+ File.delete(filename)
16
+ end
17
+ query = get_query_to_download_data_around_location(latitude, longitude, size)
18
+ text = get_overpass_query_results(query)
19
+ file = File.new(filename, 'w')
20
+ file.write text
21
+ file.close
22
+ return filename
23
+ end
24
+
25
+ def self.get_query_to_download_data_around_location(latitude, longitude, size)
26
+ min_latitude = latitude - size/2
27
+ max_latitude = latitude + size/2
28
+ min_longitude = longitude - size/2
29
+ max_longitude = longitude + size/2
30
+ bb = "#{min_latitude},#{min_longitude},#{max_latitude},#{max_longitude}"
31
+ query = '[timeout:3600];'
32
+ query += "\n"
33
+ query += "(node(#{bb});<;);"
34
+ query += "\n"
35
+ query += 'out meta;'
36
+ query += "\n"
37
+ query += '/*'
38
+ query += "\nbbox size: #{size}"
39
+ query += "\nhttp://www.openstreetmap.org/#map=17/#{latitude}/#{longitude}"
40
+ query += "\n"
41
+ query += '*/'
42
+ query += "\n"
43
+ return query
44
+ end
45
+
46
+ def self.locate_element_with_given_tags_and_type(tags, type, latitude, longitude)
47
+ max_range_in_km_for_radius = 100 #"Radius temporarly limited to 100 km to mitigate a bug.". Previously Overpass crashed for values larger than 3660
48
+
49
+ #special support for following tag values: :any_value
50
+ range = 10*1000
51
+ loop do
52
+ list = Downloader.get_overpass_query_results(Downloader.get_query_to_get_location(tags, type, latitude, longitude, range))
53
+ if list.length != 0
54
+ return self.list_returned_by_overpass_to_a_single_location(list)
55
+ end
56
+ range=range+[range, 100000].min
57
+ if range >= max_range_in_km_for_radius*1000
58
+ list = Downloader.get_overpass_query_results(Downloader.get_query_to_get_location(tags, type, latitude, longitude, :infinity))
59
+ if list.length != 0
60
+ return self.list_returned_by_overpass_to_a_single_location(list)
61
+ else
62
+ raise 'failed to find such location'
63
+ end
64
+ end
65
+ end
66
+ end
67
+
68
+ def self.list_returned_by_overpass_to_a_single_location(list)
69
+ list = list.match(/((|-)[\d\.]+)\s+((|-)[\d\.]+)/).to_a
70
+ lat = Float(list[1])
71
+ lon = Float(list[3])
72
+ return lat, lon
73
+ end
74
+
75
+ def self.get_query_to_get_location(tags, type, latitude, longitude, range)
76
+ #special support for following tag values: :any_value
77
+ locator = '[timeout:3600][out:csv(::lat,::lon;false)];'
78
+ locator += "\n"
79
+ if type == 'closed_way'
80
+ type = 'way'
81
+ end
82
+ locator += Downloader.get_query_element_to_get_location(tags, latitude, longitude, type, range)
83
+ locator +='out center;'
84
+ locator += "\n"
85
+ locator += '/*'
86
+ range_string = ''
87
+ if range == :infinity
88
+ range_string = 'infinity'
89
+ else
90
+ range_string = "#{range/1000}km"
91
+ end
92
+ locator += "\nrange: #{range_string}"
93
+ locator += "\nhttp://www.openstreetmap.org/#map=17/#{latitude}/#{longitude}"
94
+ locator += "\n"
95
+ locator += '*/'
96
+ locator += "\n"
97
+ return locator
98
+ end
99
+
100
+ def self.get_query_element_to_get_location(tags, latitude, longitude, type, range)
101
+ #special support for following tag values: :any_value
102
+ #TODO - escape value with quotation signs in them
103
+ element="(#{type}"
104
+ element += "\n"
105
+ tags.each {|tag|
106
+ if tag[1] == :any_value
107
+ element+="\t['#{tag[0]}']"
108
+ else
109
+ element+="\t['#{tag[0]}'='#{tag[1]}']"
110
+ end
111
+ element += "\n"
112
+ }
113
+ if range != :infinity
114
+ element+="\t(around:#{range},#{latitude},#{longitude});"
115
+ element += "\n"
116
+ end
117
+ element+=');'
118
+ element += "\n"
119
+ return element
120
+ end
121
+
122
+ def self.get_overpass_query_results(query, debug=false)
123
+ cached = get_overpass_query_results_from_cache(query)
124
+ return cached unless cached == nil
125
+
126
+ check_for_free_space
127
+
128
+ puts 'Running Overpass query'
129
+ if debug
130
+ puts query
131
+ puts
132
+ end
133
+ cached = Downloader.run_overpass_query query
134
+ file = File.new(get_query_cache_filename(query), 'w')
135
+ file.write cached
136
+ file.close
137
+ return cached
138
+ end
139
+
140
+ def self.get_overpass_query_results_from_cache(query)
141
+ query_cache_filename = get_query_cache_filename(query)
142
+ if File.exists?(query_cache_filename)
143
+ file = File.new(query_cache_filename)
144
+ cached = file.read
145
+ file.close
146
+ return cached
147
+ end
148
+ return nil
149
+ end
150
+
151
+ def self.get_query_cache_filename(query)
152
+ # noinspection RubyResolve
153
+ hash = Digest::SHA1.hexdigest query
154
+ query_cache_filename = CartoCSSHelper::Configuration.get_path_to_folder_for_overpass_cache + hash + '_query.cache'
155
+ return query_cache_filename
156
+ end
157
+
158
+ def self.check_for_free_space
159
+ stat = Sys::Filesystem.stat(CartoCSSHelper::Configuration.get_path_to_folder_for_overpass_cache)
160
+ gb_available = stat.block_size * stat.blocks_available / 1024 / 1024 / 1024
161
+ if gb_available < 2
162
+ #TODO cleanup cache rather than crash
163
+ raise 'less than 2GB of free space on disk with cache folder'
164
+ end
165
+ end
166
+
167
+ def self.run_overpass_query(query, retry_count=0, retry_max=5)
168
+ #default timeout is set to 180: http://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
169
+ #[timeout:3600]; should be used in all queries executed here
170
+ start = Time.now.to_s
171
+ begin
172
+ return RestClient::Request.execute(:method => :get, :url => Downloader.format_query_into_url(query), :timeout => 3600)
173
+ rescue RestClient::RequestFailed => e
174
+ puts query
175
+ puts e.response
176
+ puts e.http_code
177
+ puts start
178
+ puts Time.now.to_s
179
+ if retry_count < retry_max
180
+ sleep 60*5
181
+ Downloader.run_overpass_query(query, retry_count+1, retry_max)
182
+ else
183
+ e.raise
184
+ end
185
+ rescue ArgumentError => e
186
+ puts 'ArgumentError from rest-client, most likely caused by https://github.com/rest-client/rest-client/issues/359'
187
+ puts 'try overpass query that will return smaller amount of data'
188
+ e.raise
189
+ end
190
+ end
191
+
192
+ def self.format_query_into_url(query)
193
+ query = query.gsub(/\n/, '')
194
+ query = query.gsub(/\t/, '')
195
+ return 'http://overpass-api.de/api/interpreter?data=' + URI.escape(query)
196
+ end
197
+ end
198
+ end
@@ -0,0 +1,32 @@
1
+ require_relative 'configuration.rb'
2
+
3
+ module CartoCSSHelper
4
+ module Git
5
+ def checkout(branch, debug=false)
6
+ silence = '> /dev/null 2>&1'
7
+ if debug
8
+ silence = ''
9
+ end
10
+ Dir.chdir(Configuration.get_path_to_tilemill_project_folder) {
11
+ require 'open3'
12
+ if !system("git checkout #{branch} #{silence}")
13
+ raise 'failed checkout'
14
+ end
15
+ }
16
+ end
17
+
18
+ def get_commit_hash
19
+ Dir.chdir(Configuration.get_path_to_tilemill_project_folder) {
20
+ Open3.popen3('git log -n 1 --pretty=format:"%H"') {|_, stdout, stderr, _|
21
+ commit = stdout.read.chomp
22
+ error = stderr.read.chomp
23
+ if error != ''
24
+ raise error
25
+ end
26
+ return commit
27
+ }
28
+ }
29
+ raise 'impossible happened'
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,117 @@
1
+ module CartoCSSHelper
2
+ module Heuristic
3
+ require 'set'
4
+ require_relative 'configuration.rb'
5
+ include CartoCSSHelper::Configuration
6
+ def get_tags
7
+ tags = get_tags_from_mss
8
+ tags.merge(get_tags_from_yaml)
9
+ tags.merge(get_tags_from_osm2pqsql)
10
+ #unrecovable = Set.new [['area', 'yes'], ['area', 'no']] #TODO - how this should be handled?
11
+ #tags.merge(unrecovable)
12
+ return tags.to_a.sort
13
+ end
14
+
15
+ def get_tags_from_mss
16
+ tags = Set.new
17
+ filenames = Dir[Configuration.get_path_to_tilemill_project_folder+'*.mss']
18
+ filenames.each { |filename|
19
+ tags.merge(get_tags_from_mss_file filename)
20
+ }
21
+ return tags
22
+ end
23
+
24
+ def get_tags_from_yaml
25
+ tags = Set.new
26
+ filenames = Dir[Configuration.get_path_to_tilemill_project_folder+'*.yaml']
27
+ filenames.each { |filename|
28
+ tags.merge(get_tags_from_yaml_file filename)
29
+ }
30
+ return tags
31
+ end
32
+
33
+ def get_tags_from_osm2pqsql
34
+ tags = Set.new
35
+ filenames = Dir[Configuration.get_path_to_tilemill_project_folder+'*.style']
36
+ filenames.each { |filename|
37
+ tags.merge(get_tags_from_osm2pqsql_file filename)
38
+ }
39
+ return tags
40
+ end
41
+
42
+ def get_tags_from_mss_file(style_filename)
43
+ possible_key_values = get_tags_from_osm2pqsql
44
+ tags = Set.new
45
+ #puts style_filename
46
+ style_file = open(style_filename)
47
+ style = style_file.read
48
+ matched = style.scan(/\[feature = '(man_made|[^_]+)_([^']+)'\]/)
49
+ matched.each { |element|
50
+ key = element[0]
51
+ if possible_key_values.include?([key, get_generic_tag_value])
52
+ tags.add([key, element[1]])
53
+ end
54
+ }
55
+ matched = style.scan(/(\w+) = '(\w+)'/)
56
+ matched.each { |element|
57
+ key = element[0]
58
+ if key != 'feature'
59
+ if possible_key_values.include?([key, get_generic_tag_value])
60
+ tags.add([key, element[1]])
61
+ end
62
+ end
63
+ }
64
+ return tags
65
+ end
66
+
67
+ def get_tags_from_yaml_file(yaml_filename)
68
+ possible_key_values = get_tags_from_osm2pqsql
69
+ tags = Set.new
70
+ yaml_file = open(yaml_filename)
71
+ yaml = yaml_file.read
72
+ #(.*\.|) #WHERE p.highway = 'turning_circle'
73
+ #("|) #natural and other SQL keywords
74
+ #([^"() ]+) #tag
75
+ #repeat
76
+ # =
77
+ #'([^'()]+)' #quoted value
78
+ matched = yaml.scan(/(.*\.|)("|)([^"() ]+)("|) = '([^'()]+)'/)
79
+ matched.each { |element|
80
+ key = element[2]
81
+ value = element[4]
82
+ if possible_key_values.include?([key, get_generic_tag_value])
83
+ tags.add([key, value])
84
+ end
85
+ }
86
+ matched = yaml.scan(/("|)([^"() ]+)("|) (NOT |)IN \(([^()]*)\)/)
87
+ matched.each { |element|
88
+ tag = element[1]
89
+ key = tag.gsub(/.*\./, '')
90
+ values = element[4]
91
+ values_matched = values.scan(/'([^']+)'/)
92
+ if possible_key_values.include?([key, get_generic_tag_value])
93
+ values_matched.each { |value|
94
+ tags.add([key, value[0]])
95
+ }
96
+ end
97
+ }
98
+ return tags
99
+ end
100
+
101
+ def get_generic_tag_value
102
+ return '*'
103
+ end
104
+
105
+ def get_tags_from_osm2pqsql_file(style_filename)
106
+ tags = Set.new
107
+ #puts style_filename
108
+ style_file = open(style_filename)
109
+ style = style_file.read
110
+ matched = style.scan(/^(node,way|node|way)\s*([^ ]+)\s*text\s*($|linear|polygon|nocache)/)
111
+ matched.each { |element|
112
+ tags.add([element[1], get_generic_tag_value])
113
+ }
114
+ return tags
115
+ end
116
+ end
117
+ end