citysdk 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/Gemfile +9 -0
- data/Gemfile.lock +39 -0
- data/LICENSE.txt +22 -0
- data/README.md +52 -0
- data/Rakefile +3 -0
- data/citysdk.gemspec +29 -0
- data/lib/citysdk.rb +9 -0
- data/lib/citysdk/api.rb +216 -0
- data/lib/citysdk/file_reader.rb +387 -0
- data/lib/citysdk/importer.rb +230 -0
- data/lib/citysdk/util.rb +33 -0
- data/spec/api_spec.rb +64 -0
- data/spec/files/csvtest.zip +0 -0
- data/spec/files/geojsonTest.GeoJSON +25 -0
- data/spec/files/hotels.csv +5 -0
- data/spec/files/rk.csv +6 -0
- data/spec/files/shapeTest.zip +0 -0
- data/spec/files/stations.json +42 -0
- data/spec/files/wkb.csv +3 -0
- data/spec/import_spec.rb +137 -0
- metadata +143 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
NTM1MmNmNjc0NzJmZmVjMzNkYWMxNTVkZDcxN2Y1ZDlmMGJjM2ZlMg==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
NDgyMWU4YTFlZTExMmZiOGJkMDc5M2FkODI1N2JlZGYwYjYzOGM5Nw==
|
7
|
+
!binary "U0hBNTEy":
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
NGVjOGVkYTY3YzQ4OTJlMjczNjg3ZTliZGQxMTI4MzIwYmI5OTg1OTk5NDdi
|
10
|
+
OGVkN2NhYzQ1ZTMyMWE1M2UxYTU0MTA1MjkzNDhiZWNlZTgwYWEzOGE5MTZm
|
11
|
+
NmRhNWRlMDRhYzFlOTM5ODhmN2ZkYzcwZmNjN2M3MGI3YjliY2I=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
YzhmYmJjYjA1MzIyMjQ0MmYyNjFlZTBhZTFkOGNjYzNjM2Q2YTU2MmIxOWY0
|
14
|
+
ZmNiYjYzYmZiMWY4MjBiNjU5OTBkYzg3YWNiMThiYTIzZjZjODNiOTA0Yjg4
|
15
|
+
OGI3ZDUzM2YxNTU4ZWYwZTczOTEzZThmYjQ4MGZhYjM0MjBhZmE=
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
citysdk (0.1.0)
|
5
|
+
charlock_holmes (>= 0.6.9.4)
|
6
|
+
dbf
|
7
|
+
faraday (>= 0.8.5)
|
8
|
+
georuby (>= 2.0.0)
|
9
|
+
|
10
|
+
GEM
|
11
|
+
remote: https://rubygems.org/
|
12
|
+
specs:
|
13
|
+
charlock_holmes (0.6.9.4)
|
14
|
+
dbf (2.0.6)
|
15
|
+
fastercsv (~> 1.5.4)
|
16
|
+
diff-lcs (1.2.4)
|
17
|
+
faraday (0.8.7)
|
18
|
+
multipart-post (~> 1.1)
|
19
|
+
fastercsv (1.5.5)
|
20
|
+
georuby (2.0.0)
|
21
|
+
multipart-post (1.2.0)
|
22
|
+
rspec (2.14.0)
|
23
|
+
rspec-core (~> 2.14.0)
|
24
|
+
rspec-expectations (~> 2.14.0)
|
25
|
+
rspec-mocks (~> 2.14.0)
|
26
|
+
rspec-core (2.14.0)
|
27
|
+
rspec-expectations (2.14.0)
|
28
|
+
diff-lcs (>= 1.1.3, < 2.0)
|
29
|
+
rspec-mocks (2.14.1)
|
30
|
+
|
31
|
+
PLATFORMS
|
32
|
+
ruby
|
33
|
+
|
34
|
+
DEPENDENCIES
|
35
|
+
charlock_holmes
|
36
|
+
citysdk!
|
37
|
+
faraday
|
38
|
+
georuby
|
39
|
+
rspec
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 Waag Society
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
# Citysdk
|
2
|
+
|
3
|
+
The CitySDK gem encapulates the CItySDK API, and offers high-level file import functionalities.
|
4
|
+
The CitySDK API is part of an (open)data distribution platform developed in the EU CitySDK program by [Waag Society](http://waag.org).
|
5
|
+
Find the platform itself on [github](https://github.com/waagsociety/citysdk), platform documentation is [here](http://dev.citysdk.waag.org).
|
6
|
+
|
7
|
+
|
8
|
+
## Usage
|
9
|
+
|
10
|
+
require 'citysdk'
|
11
|
+
|
12
|
+
# check the dev site for api usage.
|
13
|
+
|
14
|
+
api = CitySDK::API.new('api.citysdk.waag.org')
|
15
|
+
|
16
|
+
# simple GET
|
17
|
+
# GET requests do not need authentication
|
18
|
+
first10layers = api.get('/layers')
|
19
|
+
puts "Number of layers: #{first10layers[:record_count]}"
|
20
|
+
puts "First layer: #{JSON.pretty_generate(first10layers[:results][0])}"
|
21
|
+
|
22
|
+
|
23
|
+
# authenticate for write actions.
|
24
|
+
exit if not api.authenticate(<email>,<passw>)
|
25
|
+
|
26
|
+
# make a layer
|
27
|
+
# when you own the 'my' top level layer domain:
|
28
|
+
@api.put('/layers',{:data => {
|
29
|
+
:name => 'my.layer',
|
30
|
+
:description => 'for testing',
|
31
|
+
:organization => 'me',
|
32
|
+
:category => 'civic.test'
|
33
|
+
}})
|
34
|
+
|
35
|
+
# add data to this layer
|
36
|
+
# attach to the node representing the city of Rotterdam
|
37
|
+
api.put('/admr.nl.rotterdam/my.layer', {:data => {:key1=>'value1', :key2=>10}})
|
38
|
+
|
39
|
+
|
40
|
+
...
|
41
|
+
|
42
|
+
# don't forget to release! this will also send 'unfilled' batches to the backend.
|
43
|
+
api.release
|
44
|
+
|
45
|
+
|
46
|
+
## Contributing
|
47
|
+
|
48
|
+
1. Fork it
|
49
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
50
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
51
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
52
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
data/citysdk.gemspec
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'citysdk'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = "citysdk"
|
8
|
+
gem.version = CitySDK::VERSION
|
9
|
+
gem.authors = ["Tom Demeyer"]
|
10
|
+
gem.email = ["tom@waag.org"]
|
11
|
+
gem.description = %q{Encapsulates the CitySDK api.}
|
12
|
+
gem.summary = %q{Encapsulates the CitySDK api, provides high-level file import functionality.}
|
13
|
+
gem.homepage = "http://citysdk.waag.org"
|
14
|
+
gem.licenses = ['MIT']
|
15
|
+
|
16
|
+
gem.files = `git ls-files`.split($/)
|
17
|
+
gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
19
|
+
gem.require_paths = ["lib"]
|
20
|
+
|
21
|
+
|
22
|
+
gem.add_dependency('dbf')
|
23
|
+
gem.add_dependency('georuby', '>= 2.0.0')
|
24
|
+
gem.add_dependency('faraday', '>= 0.8.5')
|
25
|
+
gem.add_dependency('charlock_holmes', '>= 0.6.9.4')
|
26
|
+
|
27
|
+
gem.add_development_dependency "rspec"
|
28
|
+
end
|
29
|
+
|
data/lib/citysdk.rb
ADDED
data/lib/citysdk/api.rb
ADDED
@@ -0,0 +1,216 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'faraday'
|
3
|
+
|
4
|
+
module CitySDK
|
5
|
+
|
6
|
+
class HostException < ::Exception
|
7
|
+
end
|
8
|
+
|
9
|
+
class API
|
10
|
+
attr_reader :error
|
11
|
+
attr_accessor :batch_size
|
12
|
+
|
13
|
+
@@match_tpl = {
|
14
|
+
:match => {
|
15
|
+
:params => {}
|
16
|
+
},
|
17
|
+
:nodes => []
|
18
|
+
}
|
19
|
+
@@create_tpl = {
|
20
|
+
:create => {
|
21
|
+
:params => {
|
22
|
+
:create_type => "create"
|
23
|
+
}
|
24
|
+
},
|
25
|
+
:nodes => []
|
26
|
+
}
|
27
|
+
|
28
|
+
def initialize(host, port=80)
|
29
|
+
@error = '';
|
30
|
+
@layer = '';
|
31
|
+
@batch_size = 10;
|
32
|
+
@updated = @created = 0;
|
33
|
+
set_host(host,port)
|
34
|
+
end
|
35
|
+
|
36
|
+
def authenticate(e,p)
|
37
|
+
@email = e;
|
38
|
+
@passw = p;
|
39
|
+
if !( @host == 'api.dev' or @host == 'localhost' or @host == '127.0.0.1')
|
40
|
+
auth_connection = Faraday.new :url => "https://#{@host}", :ssl => {:verify => false}
|
41
|
+
resp = auth_connection.get '/get_session', { :e => @email, :p => @passw }
|
42
|
+
else
|
43
|
+
resp = @connection.get '/get_session', { :e => @email, :p => @passw }
|
44
|
+
end
|
45
|
+
if resp.status == 200
|
46
|
+
resp = CitySDK::parseJson(resp.body)
|
47
|
+
if resp[:status] == 'success'
|
48
|
+
@connection.headers['X-Auth'] = resp[:results][0]
|
49
|
+
else
|
50
|
+
raise Exception.new(resp[:message])
|
51
|
+
end
|
52
|
+
else
|
53
|
+
raise Exception.new(resp.body)
|
54
|
+
end
|
55
|
+
|
56
|
+
if block_given?
|
57
|
+
yield
|
58
|
+
return self.release
|
59
|
+
end
|
60
|
+
true
|
61
|
+
end
|
62
|
+
|
63
|
+
def set_host(host,port=80)
|
64
|
+
@host = host
|
65
|
+
@port = port
|
66
|
+
@connection = Faraday.new :url => "http://#{@host}:#{@port}"
|
67
|
+
@connection.headers = {
|
68
|
+
:user_agent => 'CitySDK_API GEM ' + CitySDK::VERSION,
|
69
|
+
:content_type => 'application/json'
|
70
|
+
}
|
71
|
+
begin
|
72
|
+
get('/')
|
73
|
+
rescue Exception => e
|
74
|
+
raise CitySDK::Exception.new("Trouble connecting to api @ #{host}")
|
75
|
+
end
|
76
|
+
@create = @@create_tpl
|
77
|
+
@match = @@match_tpl
|
78
|
+
end
|
79
|
+
|
80
|
+
def set_matchTemplate(mtpl)
|
81
|
+
mtpl[:nodes] = []
|
82
|
+
@match = @@match_tpl = mtpl
|
83
|
+
end
|
84
|
+
|
85
|
+
def set_createTemplate(ctpl)
|
86
|
+
ctpl[:nodes] = []
|
87
|
+
@create = @@create_tpl = ctpl
|
88
|
+
end
|
89
|
+
|
90
|
+
def set_layer(l)
|
91
|
+
@layer = l
|
92
|
+
end
|
93
|
+
|
94
|
+
def set_layer_status(status)
|
95
|
+
put("/layer/#{@layer}/status",{:data => status})
|
96
|
+
end
|
97
|
+
|
98
|
+
def match_node(n)
|
99
|
+
@match[:nodes] << n
|
100
|
+
return match_flush if @match[:nodes].length >= @batch_size
|
101
|
+
return nil
|
102
|
+
end
|
103
|
+
|
104
|
+
def match_create_node(n)
|
105
|
+
@match[:nodes] << n
|
106
|
+
return match_create_flush if @match[:nodes].length >= @batch_size
|
107
|
+
end
|
108
|
+
|
109
|
+
def create_node(n)
|
110
|
+
@create[:nodes] << n
|
111
|
+
create_flush if @create[:nodes].length >= @batch_size
|
112
|
+
end
|
113
|
+
|
114
|
+
def authorized?
|
115
|
+
@connection.headers['X-Auth']
|
116
|
+
end
|
117
|
+
|
118
|
+
def release
|
119
|
+
match_flush
|
120
|
+
create_flush # send any remaining entries in the create buffer
|
121
|
+
match_create_flush
|
122
|
+
if authorized?
|
123
|
+
resp = @connection.get('/release_session')
|
124
|
+
if resp.status == 200
|
125
|
+
@connection.headers.delete('X-Auth')
|
126
|
+
else
|
127
|
+
@error = CitySDK::parseJson(resp.body)[:message]
|
128
|
+
raise HostException.new(@error)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
return [@updated, @created]
|
132
|
+
end
|
133
|
+
|
134
|
+
def delete(path)
|
135
|
+
if authorized?
|
136
|
+
resp = @connection.delete(path)
|
137
|
+
if resp.status == 200
|
138
|
+
return CitySDK::parseJson(resp.body)
|
139
|
+
end
|
140
|
+
@error = CitySDK::parseJson(resp.body)[:message]
|
141
|
+
raise HostException.new(@error)
|
142
|
+
end
|
143
|
+
raise CitySDK::Exception.new("DEL needs authorization.")
|
144
|
+
end
|
145
|
+
|
146
|
+
def post(path,data)
|
147
|
+
if authorized?
|
148
|
+
resp = @connection.post(path,data.to_json)
|
149
|
+
return CitySDK::parseJson(resp.body) if resp.status == 200
|
150
|
+
@error = CitySDK::parseJson(resp.body)[:message]
|
151
|
+
raise HostException.new(@error)
|
152
|
+
end
|
153
|
+
raise CitySDK::Exception.new("POST needs authorization.")
|
154
|
+
end
|
155
|
+
|
156
|
+
def put(path,data)
|
157
|
+
if authorized?
|
158
|
+
resp = @connection.put(path,data.to_json)
|
159
|
+
return CitySDK::parseJson(resp.body) if resp.status == 200
|
160
|
+
@error = CitySDK::parseJson(resp.body)[:message]
|
161
|
+
raise HostException.new(@error)
|
162
|
+
end
|
163
|
+
raise CitySDK::Exception.new("PUT needs authorization.")
|
164
|
+
end
|
165
|
+
|
166
|
+
def get(path)
|
167
|
+
resp = @connection.get(path)
|
168
|
+
return CitySDK::parseJson(resp.body) if resp.status == 200
|
169
|
+
@error = CitySDK::parseJson(resp.body)[:message]
|
170
|
+
raise HostException.new(@error)
|
171
|
+
end
|
172
|
+
|
173
|
+
def match_create_flush
|
174
|
+
|
175
|
+
if @match[:nodes].length > 0
|
176
|
+
resp = post('util/match',@match)
|
177
|
+
if resp[:nodes].length > 0
|
178
|
+
@create[:nodes] = resp[:nodes]
|
179
|
+
res = put("/nodes/#{@layer}",@create)
|
180
|
+
tally(res)
|
181
|
+
@create[:nodes] = []
|
182
|
+
end
|
183
|
+
@match[:nodes] = []
|
184
|
+
res
|
185
|
+
end
|
186
|
+
nil
|
187
|
+
end
|
188
|
+
|
189
|
+
def match_flush
|
190
|
+
if @match[:nodes].length > 0
|
191
|
+
resp = post('util/match',@match)
|
192
|
+
@match[:nodes] = []
|
193
|
+
return resp
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
|
198
|
+
def create_flush
|
199
|
+
if @create[:nodes].length > 0
|
200
|
+
tally put("/nodes/#{@layer}",@create)
|
201
|
+
@create[:nodes] = []
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
def tally(res)
|
206
|
+
if res[:status] == "success"
|
207
|
+
# TODO: also tally debug data!
|
208
|
+
@updated += res[:create][:results][:totals][:updated]
|
209
|
+
@created += res[:create][:results][:totals][:created]
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
end
|
214
|
+
|
215
|
+
end
|
216
|
+
|
@@ -0,0 +1,387 @@
|
|
1
|
+
require 'csv'
|
2
|
+
require 'geo_ruby'
|
3
|
+
require 'geo_ruby/shp'
|
4
|
+
require 'geo_ruby/geojson'
|
5
|
+
require 'charlock_holmes'
|
6
|
+
require 'tmpdir'
|
7
|
+
|
8
|
+
module CitySDK
|
9
|
+
|
10
|
+
|
11
|
+
class FileReader
|
12
|
+
|
13
|
+
RE_NAME = /(title|titel|naam|name)/i
|
14
|
+
RE_A_NAME = /^(naam|name|title|titel)$/i
|
15
|
+
RE_GEO = /^geom(etry)?|location|locatie$/i
|
16
|
+
RE_Y = /lat|(y.*coord)|(y.*loc(atie|ation)?)/i
|
17
|
+
RE_X = /lon|lng|(x.*coord)|(x.*loc(atie|ation)?)/i
|
18
|
+
|
19
|
+
attr_reader :file, :content,:params
|
20
|
+
|
21
|
+
def initialize(pars)
|
22
|
+
@params = pars
|
23
|
+
file_path = File.expand_path(@params[:file_path])
|
24
|
+
case File.extname(file_path)
|
25
|
+
when /\.zip/i
|
26
|
+
readZip(file_path)
|
27
|
+
when /\.(geo)?json/i
|
28
|
+
readJSON(file_path)
|
29
|
+
when /\.shape/i
|
30
|
+
readShape(file_path)
|
31
|
+
when /\.csv|tsv/i
|
32
|
+
readCsv(file_path)
|
33
|
+
when /\.csdk/i
|
34
|
+
readCsdk(file_path)
|
35
|
+
end
|
36
|
+
|
37
|
+
@params[:rowcount] = @content.length
|
38
|
+
getFields if not @params[:fields]
|
39
|
+
guessName if not @params[:name]
|
40
|
+
guessSRID if not @params[:srid]
|
41
|
+
findUniqueField if not @params[:unique_id]
|
42
|
+
getAddress if not @params[:hasaddress]
|
43
|
+
@params[:hasgeometry] = 'unknown' if @params[:hasgeometry].nil?
|
44
|
+
end
|
45
|
+
|
46
|
+
def getAddress
|
47
|
+
pd = pc = hn = ad = false
|
48
|
+
@params[:fields].reverse.each do |f|
|
49
|
+
pd = f if ( f.to_s =~ /postcode|post/i )
|
50
|
+
pc = f if ( f.to_s =~ /^postcode$/i )
|
51
|
+
hn = f if ( f.to_s =~ /huisnummer|housenumber|(house|huis)(nr|no)|number/i)
|
52
|
+
ad = f if ( f.to_s =~ /address|street|straat|adres/i)
|
53
|
+
end
|
54
|
+
@params[:hasaddress] = 'unknown'
|
55
|
+
if (ad or hn)
|
56
|
+
if pc
|
57
|
+
@params[:hasaddress] = 'certain'
|
58
|
+
@params[:postcode] = pc
|
59
|
+
elsif pd
|
60
|
+
@params[:hasaddress] = 'maybe'
|
61
|
+
@params[:postcode] = pd
|
62
|
+
end
|
63
|
+
@params[:housenumber] = hn ? hn : ad
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def findUniqueField
|
68
|
+
fields = {}
|
69
|
+
unfield = nil
|
70
|
+
|
71
|
+
return if @content[0][:id]
|
72
|
+
|
73
|
+
@content.each do |h|
|
74
|
+
h[:properties].each do |k,v|
|
75
|
+
fields[k] = {} if fields[k].nil?
|
76
|
+
(fields[k][v] == nil) ? fields[k][v] = 1 : fields[k][v] += 1
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
fields.each_key do |k|
|
81
|
+
if fields[k].length == @params[:rowcount]
|
82
|
+
@params[:unique_id] = unfield = k
|
83
|
+
break
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
if unfield
|
88
|
+
@content.each do |h|
|
89
|
+
h[:id] = h[:properties][unfield]
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
94
|
+
|
95
|
+
def guessName
|
96
|
+
@params[:fields].reverse.each do |k|
|
97
|
+
if(k.to_s =~ RE_A_NAME)
|
98
|
+
@params[:name] = k
|
99
|
+
return
|
100
|
+
end
|
101
|
+
if(k.to_s =~ RE_NAME)
|
102
|
+
@params[:name] = k
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def getFields
|
108
|
+
@params[:fields] = []
|
109
|
+
@content[0][:properties].each_key do |k|
|
110
|
+
@params[:fields] << k
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def guessSRID
|
115
|
+
return if @content[0][:geometry].nil?
|
116
|
+
@params[:srid] = 4326
|
117
|
+
g = @content[0][:geometry][:coordinates]
|
118
|
+
while g[0].is_a?(Array)
|
119
|
+
g = g[0]
|
120
|
+
end
|
121
|
+
lon = g[0]
|
122
|
+
lat = g[1]
|
123
|
+
# if lon > -180.0 and lon < 180.0 and lat > -90.0 and lat < 90.0
|
124
|
+
# @params[:srid] = 4326
|
125
|
+
# els
|
126
|
+
if lon > -7000.0 and lon < 300000.0 and lat > 289000.0 and lat < 629000.0
|
127
|
+
# Dutch new rd system
|
128
|
+
@params[:srid] = 28992
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def findColSep(f)
|
133
|
+
a = f.gets
|
134
|
+
b = f.gets
|
135
|
+
[";","\t","|"].each do |s|
|
136
|
+
return s if (a.split(s).length == b.split(s).length) and b.split(s).length > 1
|
137
|
+
end
|
138
|
+
','
|
139
|
+
end
|
140
|
+
|
141
|
+
def isWkbGeometry(s)
|
142
|
+
begin
|
143
|
+
f = GeoRuby::SimpleFeatures::GeometryFactory::new
|
144
|
+
p = GeoRuby::SimpleFeatures::HexEWKBParser.new(f)
|
145
|
+
p.parse(s)
|
146
|
+
g = f.geometry
|
147
|
+
return g.srid,g.as_json[:type],g
|
148
|
+
rescue Exception=>e
|
149
|
+
end
|
150
|
+
nil
|
151
|
+
end
|
152
|
+
|
153
|
+
def isGeoJSON(s)
|
154
|
+
begin
|
155
|
+
if ['Point', 'MultiPoint', 'LineString', 'MultiLineString', 'Polygon', 'MultiPolygon', 'GeometryCollection'].include?(s[:type])
|
156
|
+
srid = 4326
|
157
|
+
if s[:crs] && s[:crs][:type] == 'OGC'
|
158
|
+
urn = s[:crs][:properties][:urn].split(':')
|
159
|
+
srid = urn.last.to_i if (urn[4] == 'EPSG')
|
160
|
+
end
|
161
|
+
return srid,s[:type],s
|
162
|
+
end
|
163
|
+
rescue Exception=>e
|
164
|
+
end
|
165
|
+
nil
|
166
|
+
end
|
167
|
+
|
168
|
+
|
169
|
+
def findGeometry
|
170
|
+
xfield = nil; xs = true
|
171
|
+
yfield = nil; ys = true
|
172
|
+
@content[0][:properties].each do |k,v|
|
173
|
+
if k.to_s =~ RE_GEO
|
174
|
+
srid,g_type = isWkbGeometry(v)
|
175
|
+
if(srid)
|
176
|
+
@params[:srid] = srid
|
177
|
+
@params[:geomtry_type] = g_type
|
178
|
+
@content.each do |h|
|
179
|
+
a,b,g = isWkbGeometry(h[:properties][k])
|
180
|
+
h[:geometry] = g
|
181
|
+
h[:properties].delete(k)
|
182
|
+
end
|
183
|
+
@params[:hasgeometry] = 'certain'
|
184
|
+
return true
|
185
|
+
end
|
186
|
+
|
187
|
+
srid,g_type = isGeoJSON(v)
|
188
|
+
if(srid)
|
189
|
+
@params[:srid] = srid
|
190
|
+
@params[:geomtry_type] = g_type
|
191
|
+
@content.each do |h|
|
192
|
+
h[:geometry] = h[:properties][k]
|
193
|
+
h[:properties].delete(k)
|
194
|
+
end
|
195
|
+
@params[:hasgeometry] = 'certain'
|
196
|
+
return true
|
197
|
+
end
|
198
|
+
|
199
|
+
@content.each do |h|
|
200
|
+
h[:geometry] = h[:properties][k]
|
201
|
+
h[:properties].delete(k)
|
202
|
+
end
|
203
|
+
@params[:hasgeometry] = 'maybe'
|
204
|
+
return
|
205
|
+
end
|
206
|
+
|
207
|
+
hdc = k.to_sym.downcase
|
208
|
+
if hdc == 'longitude' or hdc == 'lon'
|
209
|
+
xfield=k; xs=false
|
210
|
+
end
|
211
|
+
if hdc == 'latitude' or hdc == 'lat'
|
212
|
+
yfield=k; ys=false
|
213
|
+
end
|
214
|
+
xfield = k if xs and (hdc =~ RE_X)
|
215
|
+
yfield = k if ys and (hdc =~ RE_Y)
|
216
|
+
end
|
217
|
+
|
218
|
+
if xfield and yfield and (xfield != yfield)
|
219
|
+
@params[:hasgeometry] = 'certain'
|
220
|
+
@content.each do |h|
|
221
|
+
h[:geometry] = {:type => 'Point', :coordinates => [h[:properties][xfield].gsub(',','.').to_f, h[:properties][yfield].gsub(',','.').to_f]}
|
222
|
+
h[:properties].delete(yfield)
|
223
|
+
h[:properties].delete(xfield)
|
224
|
+
end
|
225
|
+
@params[:geomtry_type] = 'Point'
|
226
|
+
return true
|
227
|
+
end
|
228
|
+
false
|
229
|
+
end
|
230
|
+
|
231
|
+
|
232
|
+
def readCsv(path)
|
233
|
+
@file = path
|
234
|
+
c=''
|
235
|
+
File.open(path, "r:bom|utf-8") do |fd|
|
236
|
+
c = fd.read
|
237
|
+
end
|
238
|
+
if true != @params[:utf8_fixed]
|
239
|
+
detect = CharlockHolmes::EncodingDetector.detect(c)
|
240
|
+
c = CharlockHolmes::Converter.convert(c, detect[:encoding], 'UTF-8') if detect
|
241
|
+
end
|
242
|
+
c = c.force_encoding('utf-8')
|
243
|
+
@content = []
|
244
|
+
@params[:colsep] = findColSep(StringIO.new(c))
|
245
|
+
csv = CSV.new(c, :col_sep => @params[:colsep], :headers => true, :skip_blanks =>true)
|
246
|
+
csv.each do |row|
|
247
|
+
r = row.to_hash
|
248
|
+
h = {}
|
249
|
+
r.each do |k,v|
|
250
|
+
h[(k.to_sym rescue k) || k] = v
|
251
|
+
end
|
252
|
+
@content << {:properties => h }
|
253
|
+
end
|
254
|
+
findGeometry
|
255
|
+
end
|
256
|
+
|
257
|
+
|
258
|
+
def readJSON(path)
|
259
|
+
@content = []
|
260
|
+
@file = path
|
261
|
+
raw = ''
|
262
|
+
File.open(path, "r:bom|utf-8") do |fd|
|
263
|
+
raw = fd.read
|
264
|
+
end
|
265
|
+
hash = CitySDK::parseJson(raw)
|
266
|
+
|
267
|
+
if hash.is_a?(Hash) and hash[:type] and (hash[:type] == 'FeatureCollection')
|
268
|
+
hash[:features].each do |f|
|
269
|
+
f.delete(:type)
|
270
|
+
@content << f
|
271
|
+
end
|
272
|
+
@params[:hasgeometry] = 'certain'
|
273
|
+
findUniqueField if @content[0][:id].nil?
|
274
|
+
else
|
275
|
+
val,length = nil,0
|
276
|
+
|
277
|
+
if hash.is_a?(Array)
|
278
|
+
val,length = hash,hash.length
|
279
|
+
else
|
280
|
+
hash.each do |k,v|
|
281
|
+
if v.is_a?(Array)
|
282
|
+
val,length = v,v.length if v.length > length
|
283
|
+
end
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
if val
|
288
|
+
val.each do |h|
|
289
|
+
@content << {:properties => h}
|
290
|
+
end
|
291
|
+
end
|
292
|
+
findGeometry
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
|
297
|
+
def sridFromPrj(str)
|
298
|
+
begin
|
299
|
+
connection = Faraday.new :url => "http://prj2epsg.org"
|
300
|
+
resp = connection.get('/search.json', {:mode => 'wkt', :terms => str})
|
301
|
+
if resp.status == 200
|
302
|
+
resp = CitySDK::parseJson resp.body
|
303
|
+
@params[:srid] = resp[:codes][0][:code].to_i
|
304
|
+
end
|
305
|
+
rescue
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
def readShape(path)
|
310
|
+
@content = []
|
311
|
+
@file = path
|
312
|
+
|
313
|
+
prj = path.gsub(/.shp$/i,"") + '.prj'
|
314
|
+
prj = File.exists?(prj) ? File.read(prj) : nil
|
315
|
+
sridFromPrj(prj) if (prj and @params[:srid].nil?)
|
316
|
+
|
317
|
+
@params[:hasgeometry] = 'certain'
|
318
|
+
|
319
|
+
GeoRuby::Shp4r::ShpFile.open(path) do |shp|
|
320
|
+
shp.each do |shape|
|
321
|
+
h = {}
|
322
|
+
h[:geometry] = CitySDK::parseJson(shape.geometry.to_json) #a GeoRuby SimpleFeature
|
323
|
+
h[:properties] = {}
|
324
|
+
att_data = shape.data #a Hash
|
325
|
+
shp.fields.each do |field|
|
326
|
+
h[:properties][field.name.to_sym] = att_data[field.name]
|
327
|
+
end
|
328
|
+
@content << h
|
329
|
+
end
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
def readCsdk(path)
|
334
|
+
h = Marshal.load(File.read(path))
|
335
|
+
@params = h[:config]
|
336
|
+
@content = h[:content]
|
337
|
+
end
|
338
|
+
|
339
|
+
def readZip(path)
|
340
|
+
begin
|
341
|
+
Dir.mktmpdir("cdkfi_#{File.basename(path)}") do |dir|
|
342
|
+
raise "Error unzipping #{path}." if not system "unzip #{path} -d #{dir} > /dev/null 2>&1"
|
343
|
+
Dir.foreach(dir) do |f|
|
344
|
+
next if f =~ /^\./
|
345
|
+
case File.extname(f)
|
346
|
+
when /\.(geo)?json/i
|
347
|
+
readJSON(dir+'/'+f)
|
348
|
+
return
|
349
|
+
when /\.shp/i
|
350
|
+
readShape(dir+'/'+f)
|
351
|
+
return
|
352
|
+
when /\.csv|tsv/i
|
353
|
+
readCsv(dir+'/'+f)
|
354
|
+
return
|
355
|
+
end
|
356
|
+
end
|
357
|
+
end
|
358
|
+
rescue Exception => e
|
359
|
+
raise CitySDK::Exception(e.message, {:originalfile => path}, __FILE__,__LINE__)
|
360
|
+
end
|
361
|
+
end
|
362
|
+
|
363
|
+
def write(path=nil)
|
364
|
+
begin
|
365
|
+
path = @file_path if path.nil?
|
366
|
+
File.open(path+'.csdk',"w") do |fd|
|
367
|
+
fd.write( Marshal.dump({:config=>@params, :content=>@content}) )
|
368
|
+
# fd.write(JSON.pretty_generate({:config=>@params, :content=>@content}))
|
369
|
+
end
|
370
|
+
rescue
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
374
|
+
end
|
375
|
+
|
376
|
+
end
|
377
|
+
|
378
|
+
|
379
|
+
|
380
|
+
# {
|
381
|
+
# :headers => ['aap','noot','titel', 'gid', 'geometry']
|
382
|
+
# :config => {:geom => 'geometry', :name => 'titel', :id => 'gid', :srid => 28892}
|
383
|
+
# :data => [
|
384
|
+
# ['jpo','pipo','naam1','1092',{:type => 'Point', :coordinates => [5.3, 52.4]}],
|
385
|
+
# ['jpa','popi','naam2','1093',{:type => 'Point', :coordinates => [5.1, 52.1]}]
|
386
|
+
# ]
|
387
|
+
# }
|