citysdk 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/Gemfile +9 -0
- data/Gemfile.lock +39 -0
- data/LICENSE.txt +22 -0
- data/README.md +52 -0
- data/Rakefile +3 -0
- data/citysdk.gemspec +29 -0
- data/lib/citysdk.rb +9 -0
- data/lib/citysdk/api.rb +216 -0
- data/lib/citysdk/file_reader.rb +387 -0
- data/lib/citysdk/importer.rb +230 -0
- data/lib/citysdk/util.rb +33 -0
- data/spec/api_spec.rb +64 -0
- data/spec/files/csvtest.zip +0 -0
- data/spec/files/geojsonTest.GeoJSON +25 -0
- data/spec/files/hotels.csv +5 -0
- data/spec/files/rk.csv +6 -0
- data/spec/files/shapeTest.zip +0 -0
- data/spec/files/stations.json +42 -0
- data/spec/files/wkb.csv +3 -0
- data/spec/import_spec.rb +137 -0
- metadata +143 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
NTM1MmNmNjc0NzJmZmVjMzNkYWMxNTVkZDcxN2Y1ZDlmMGJjM2ZlMg==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
NDgyMWU4YTFlZTExMmZiOGJkMDc5M2FkODI1N2JlZGYwYjYzOGM5Nw==
|
7
|
+
!binary "U0hBNTEy":
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
NGVjOGVkYTY3YzQ4OTJlMjczNjg3ZTliZGQxMTI4MzIwYmI5OTg1OTk5NDdi
|
10
|
+
OGVkN2NhYzQ1ZTMyMWE1M2UxYTU0MTA1MjkzNDhiZWNlZTgwYWEzOGE5MTZm
|
11
|
+
NmRhNWRlMDRhYzFlOTM5ODhmN2ZkYzcwZmNjN2M3MGI3YjliY2I=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
YzhmYmJjYjA1MzIyMjQ0MmYyNjFlZTBhZTFkOGNjYzNjM2Q2YTU2MmIxOWY0
|
14
|
+
ZmNiYjYzYmZiMWY4MjBiNjU5OTBkYzg3YWNiMThiYTIzZjZjODNiOTA0Yjg4
|
15
|
+
OGI3ZDUzM2YxNTU4ZWYwZTczOTEzZThmYjQ4MGZhYjM0MjBhZmE=
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
citysdk (0.1.0)
|
5
|
+
charlock_holmes (>= 0.6.9.4)
|
6
|
+
dbf
|
7
|
+
faraday (>= 0.8.5)
|
8
|
+
georuby (>= 2.0.0)
|
9
|
+
|
10
|
+
GEM
|
11
|
+
remote: https://rubygems.org/
|
12
|
+
specs:
|
13
|
+
charlock_holmes (0.6.9.4)
|
14
|
+
dbf (2.0.6)
|
15
|
+
fastercsv (~> 1.5.4)
|
16
|
+
diff-lcs (1.2.4)
|
17
|
+
faraday (0.8.7)
|
18
|
+
multipart-post (~> 1.1)
|
19
|
+
fastercsv (1.5.5)
|
20
|
+
georuby (2.0.0)
|
21
|
+
multipart-post (1.2.0)
|
22
|
+
rspec (2.14.0)
|
23
|
+
rspec-core (~> 2.14.0)
|
24
|
+
rspec-expectations (~> 2.14.0)
|
25
|
+
rspec-mocks (~> 2.14.0)
|
26
|
+
rspec-core (2.14.0)
|
27
|
+
rspec-expectations (2.14.0)
|
28
|
+
diff-lcs (>= 1.1.3, < 2.0)
|
29
|
+
rspec-mocks (2.14.1)
|
30
|
+
|
31
|
+
PLATFORMS
|
32
|
+
ruby
|
33
|
+
|
34
|
+
DEPENDENCIES
|
35
|
+
charlock_holmes
|
36
|
+
citysdk!
|
37
|
+
faraday
|
38
|
+
georuby
|
39
|
+
rspec
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 Waag Society
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
# Citysdk
|
2
|
+
|
3
|
+
The CitySDK gem encapulates the CItySDK API, and offers high-level file import functionalities.
|
4
|
+
The CitySDK API is part of an (open)data distribution platform developed in the EU CitySDK program by [Waag Society](http://waag.org).
|
5
|
+
Find the platform itself on [github](https://github.com/waagsociety/citysdk), platform documentation is [here](http://dev.citysdk.waag.org).
|
6
|
+
|
7
|
+
|
8
|
+
## Usage
|
9
|
+
|
10
|
+
require 'citysdk'
|
11
|
+
|
12
|
+
# check the dev site for api usage.
|
13
|
+
|
14
|
+
api = CitySDK::API.new('api.citysdk.waag.org')
|
15
|
+
|
16
|
+
# simple GET
|
17
|
+
# GET requests do not need authentication
|
18
|
+
first10layers = api.get('/layers')
|
19
|
+
puts "Number of layers: #{first10layers[:record_count]}"
|
20
|
+
puts "First layer: #{JSON.pretty_generate(first10layers[:results][0])}"
|
21
|
+
|
22
|
+
|
23
|
+
# authenticate for write actions.
|
24
|
+
exit if not api.authenticate(<email>,<passw>)
|
25
|
+
|
26
|
+
# make a layer
|
27
|
+
# when you own the 'my' top level layer domain:
|
28
|
+
@api.put('/layers',{:data => {
|
29
|
+
:name => 'my.layer',
|
30
|
+
:description => 'for testing',
|
31
|
+
:organization => 'me',
|
32
|
+
:category => 'civic.test'
|
33
|
+
}})
|
34
|
+
|
35
|
+
# add data to this layer
|
36
|
+
# attach to the node representing the city of Rotterdam
|
37
|
+
api.put('/admr.nl.rotterdam/my.layer', {:data => {:key1=>'value1', :key2=>10}})
|
38
|
+
|
39
|
+
|
40
|
+
...
|
41
|
+
|
42
|
+
# don't forget to release! this will also send 'unfilled' batches to the backend.
|
43
|
+
api.release
|
44
|
+
|
45
|
+
|
46
|
+
## Contributing
|
47
|
+
|
48
|
+
1. Fork it
|
49
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
50
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
51
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
52
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
data/citysdk.gemspec
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'citysdk'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = "citysdk"
|
8
|
+
gem.version = CitySDK::VERSION
|
9
|
+
gem.authors = ["Tom Demeyer"]
|
10
|
+
gem.email = ["tom@waag.org"]
|
11
|
+
gem.description = %q{Encapsulates the CitySDK api.}
|
12
|
+
gem.summary = %q{Encapsulates the CitySDK api, provides high-level file import functionality.}
|
13
|
+
gem.homepage = "http://citysdk.waag.org"
|
14
|
+
gem.licenses = ['MIT']
|
15
|
+
|
16
|
+
gem.files = `git ls-files`.split($/)
|
17
|
+
gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
19
|
+
gem.require_paths = ["lib"]
|
20
|
+
|
21
|
+
|
22
|
+
gem.add_dependency('dbf')
|
23
|
+
gem.add_dependency('georuby', '>= 2.0.0')
|
24
|
+
gem.add_dependency('faraday', '>= 0.8.5')
|
25
|
+
gem.add_dependency('charlock_holmes', '>= 0.6.9.4')
|
26
|
+
|
27
|
+
gem.add_development_dependency "rspec"
|
28
|
+
end
|
29
|
+
|
data/lib/citysdk.rb
ADDED
data/lib/citysdk/api.rb
ADDED
@@ -0,0 +1,216 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'faraday'
|
3
|
+
|
4
|
+
module CitySDK
|
5
|
+
|
6
|
+
class HostException < ::Exception
|
7
|
+
end
|
8
|
+
|
9
|
+
class API
|
10
|
+
attr_reader :error
|
11
|
+
attr_accessor :batch_size
|
12
|
+
|
13
|
+
@@match_tpl = {
|
14
|
+
:match => {
|
15
|
+
:params => {}
|
16
|
+
},
|
17
|
+
:nodes => []
|
18
|
+
}
|
19
|
+
@@create_tpl = {
|
20
|
+
:create => {
|
21
|
+
:params => {
|
22
|
+
:create_type => "create"
|
23
|
+
}
|
24
|
+
},
|
25
|
+
:nodes => []
|
26
|
+
}
|
27
|
+
|
28
|
+
def initialize(host, port=80)
|
29
|
+
@error = '';
|
30
|
+
@layer = '';
|
31
|
+
@batch_size = 10;
|
32
|
+
@updated = @created = 0;
|
33
|
+
set_host(host,port)
|
34
|
+
end
|
35
|
+
|
36
|
+
def authenticate(e,p)
|
37
|
+
@email = e;
|
38
|
+
@passw = p;
|
39
|
+
if !( @host == 'api.dev' or @host == 'localhost' or @host == '127.0.0.1')
|
40
|
+
auth_connection = Faraday.new :url => "https://#{@host}", :ssl => {:verify => false}
|
41
|
+
resp = auth_connection.get '/get_session', { :e => @email, :p => @passw }
|
42
|
+
else
|
43
|
+
resp = @connection.get '/get_session', { :e => @email, :p => @passw }
|
44
|
+
end
|
45
|
+
if resp.status == 200
|
46
|
+
resp = CitySDK::parseJson(resp.body)
|
47
|
+
if resp[:status] == 'success'
|
48
|
+
@connection.headers['X-Auth'] = resp[:results][0]
|
49
|
+
else
|
50
|
+
raise Exception.new(resp[:message])
|
51
|
+
end
|
52
|
+
else
|
53
|
+
raise Exception.new(resp.body)
|
54
|
+
end
|
55
|
+
|
56
|
+
if block_given?
|
57
|
+
yield
|
58
|
+
return self.release
|
59
|
+
end
|
60
|
+
true
|
61
|
+
end
|
62
|
+
|
63
|
+
def set_host(host,port=80)
|
64
|
+
@host = host
|
65
|
+
@port = port
|
66
|
+
@connection = Faraday.new :url => "http://#{@host}:#{@port}"
|
67
|
+
@connection.headers = {
|
68
|
+
:user_agent => 'CitySDK_API GEM ' + CitySDK::VERSION,
|
69
|
+
:content_type => 'application/json'
|
70
|
+
}
|
71
|
+
begin
|
72
|
+
get('/')
|
73
|
+
rescue Exception => e
|
74
|
+
raise CitySDK::Exception.new("Trouble connecting to api @ #{host}")
|
75
|
+
end
|
76
|
+
@create = @@create_tpl
|
77
|
+
@match = @@match_tpl
|
78
|
+
end
|
79
|
+
|
80
|
+
def set_matchTemplate(mtpl)
|
81
|
+
mtpl[:nodes] = []
|
82
|
+
@match = @@match_tpl = mtpl
|
83
|
+
end
|
84
|
+
|
85
|
+
def set_createTemplate(ctpl)
|
86
|
+
ctpl[:nodes] = []
|
87
|
+
@create = @@create_tpl = ctpl
|
88
|
+
end
|
89
|
+
|
90
|
+
def set_layer(l)
|
91
|
+
@layer = l
|
92
|
+
end
|
93
|
+
|
94
|
+
def set_layer_status(status)
|
95
|
+
put("/layer/#{@layer}/status",{:data => status})
|
96
|
+
end
|
97
|
+
|
98
|
+
def match_node(n)
|
99
|
+
@match[:nodes] << n
|
100
|
+
return match_flush if @match[:nodes].length >= @batch_size
|
101
|
+
return nil
|
102
|
+
end
|
103
|
+
|
104
|
+
def match_create_node(n)
|
105
|
+
@match[:nodes] << n
|
106
|
+
return match_create_flush if @match[:nodes].length >= @batch_size
|
107
|
+
end
|
108
|
+
|
109
|
+
def create_node(n)
|
110
|
+
@create[:nodes] << n
|
111
|
+
create_flush if @create[:nodes].length >= @batch_size
|
112
|
+
end
|
113
|
+
|
114
|
+
def authorized?
|
115
|
+
@connection.headers['X-Auth']
|
116
|
+
end
|
117
|
+
|
118
|
+
def release
|
119
|
+
match_flush
|
120
|
+
create_flush # send any remaining entries in the create buffer
|
121
|
+
match_create_flush
|
122
|
+
if authorized?
|
123
|
+
resp = @connection.get('/release_session')
|
124
|
+
if resp.status == 200
|
125
|
+
@connection.headers.delete('X-Auth')
|
126
|
+
else
|
127
|
+
@error = CitySDK::parseJson(resp.body)[:message]
|
128
|
+
raise HostException.new(@error)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
return [@updated, @created]
|
132
|
+
end
|
133
|
+
|
134
|
+
def delete(path)
|
135
|
+
if authorized?
|
136
|
+
resp = @connection.delete(path)
|
137
|
+
if resp.status == 200
|
138
|
+
return CitySDK::parseJson(resp.body)
|
139
|
+
end
|
140
|
+
@error = CitySDK::parseJson(resp.body)[:message]
|
141
|
+
raise HostException.new(@error)
|
142
|
+
end
|
143
|
+
raise CitySDK::Exception.new("DEL needs authorization.")
|
144
|
+
end
|
145
|
+
|
146
|
+
def post(path,data)
|
147
|
+
if authorized?
|
148
|
+
resp = @connection.post(path,data.to_json)
|
149
|
+
return CitySDK::parseJson(resp.body) if resp.status == 200
|
150
|
+
@error = CitySDK::parseJson(resp.body)[:message]
|
151
|
+
raise HostException.new(@error)
|
152
|
+
end
|
153
|
+
raise CitySDK::Exception.new("POST needs authorization.")
|
154
|
+
end
|
155
|
+
|
156
|
+
def put(path,data)
|
157
|
+
if authorized?
|
158
|
+
resp = @connection.put(path,data.to_json)
|
159
|
+
return CitySDK::parseJson(resp.body) if resp.status == 200
|
160
|
+
@error = CitySDK::parseJson(resp.body)[:message]
|
161
|
+
raise HostException.new(@error)
|
162
|
+
end
|
163
|
+
raise CitySDK::Exception.new("PUT needs authorization.")
|
164
|
+
end
|
165
|
+
|
166
|
+
def get(path)
|
167
|
+
resp = @connection.get(path)
|
168
|
+
return CitySDK::parseJson(resp.body) if resp.status == 200
|
169
|
+
@error = CitySDK::parseJson(resp.body)[:message]
|
170
|
+
raise HostException.new(@error)
|
171
|
+
end
|
172
|
+
|
173
|
+
def match_create_flush
|
174
|
+
|
175
|
+
if @match[:nodes].length > 0
|
176
|
+
resp = post('util/match',@match)
|
177
|
+
if resp[:nodes].length > 0
|
178
|
+
@create[:nodes] = resp[:nodes]
|
179
|
+
res = put("/nodes/#{@layer}",@create)
|
180
|
+
tally(res)
|
181
|
+
@create[:nodes] = []
|
182
|
+
end
|
183
|
+
@match[:nodes] = []
|
184
|
+
res
|
185
|
+
end
|
186
|
+
nil
|
187
|
+
end
|
188
|
+
|
189
|
+
def match_flush
|
190
|
+
if @match[:nodes].length > 0
|
191
|
+
resp = post('util/match',@match)
|
192
|
+
@match[:nodes] = []
|
193
|
+
return resp
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
|
198
|
+
def create_flush
|
199
|
+
if @create[:nodes].length > 0
|
200
|
+
tally put("/nodes/#{@layer}",@create)
|
201
|
+
@create[:nodes] = []
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
def tally(res)
|
206
|
+
if res[:status] == "success"
|
207
|
+
# TODO: also tally debug data!
|
208
|
+
@updated += res[:create][:results][:totals][:updated]
|
209
|
+
@created += res[:create][:results][:totals][:created]
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
end
|
214
|
+
|
215
|
+
end
|
216
|
+
|
@@ -0,0 +1,387 @@
|
|
1
|
+
require 'csv'
|
2
|
+
require 'geo_ruby'
|
3
|
+
require 'geo_ruby/shp'
|
4
|
+
require 'geo_ruby/geojson'
|
5
|
+
require 'charlock_holmes'
|
6
|
+
require 'tmpdir'
|
7
|
+
|
8
|
+
module CitySDK
|
9
|
+
|
10
|
+
|
11
|
+
class FileReader
|
12
|
+
|
13
|
+
RE_NAME = /(title|titel|naam|name)/i
|
14
|
+
RE_A_NAME = /^(naam|name|title|titel)$/i
|
15
|
+
RE_GEO = /^geom(etry)?|location|locatie$/i
|
16
|
+
RE_Y = /lat|(y.*coord)|(y.*loc(atie|ation)?)/i
|
17
|
+
RE_X = /lon|lng|(x.*coord)|(x.*loc(atie|ation)?)/i
|
18
|
+
|
19
|
+
attr_reader :file, :content,:params
|
20
|
+
|
21
|
+
def initialize(pars)
|
22
|
+
@params = pars
|
23
|
+
file_path = File.expand_path(@params[:file_path])
|
24
|
+
case File.extname(file_path)
|
25
|
+
when /\.zip/i
|
26
|
+
readZip(file_path)
|
27
|
+
when /\.(geo)?json/i
|
28
|
+
readJSON(file_path)
|
29
|
+
when /\.shape/i
|
30
|
+
readShape(file_path)
|
31
|
+
when /\.csv|tsv/i
|
32
|
+
readCsv(file_path)
|
33
|
+
when /\.csdk/i
|
34
|
+
readCsdk(file_path)
|
35
|
+
end
|
36
|
+
|
37
|
+
@params[:rowcount] = @content.length
|
38
|
+
getFields if not @params[:fields]
|
39
|
+
guessName if not @params[:name]
|
40
|
+
guessSRID if not @params[:srid]
|
41
|
+
findUniqueField if not @params[:unique_id]
|
42
|
+
getAddress if not @params[:hasaddress]
|
43
|
+
@params[:hasgeometry] = 'unknown' if @params[:hasgeometry].nil?
|
44
|
+
end
|
45
|
+
|
46
|
+
def getAddress
|
47
|
+
pd = pc = hn = ad = false
|
48
|
+
@params[:fields].reverse.each do |f|
|
49
|
+
pd = f if ( f.to_s =~ /postcode|post/i )
|
50
|
+
pc = f if ( f.to_s =~ /^postcode$/i )
|
51
|
+
hn = f if ( f.to_s =~ /huisnummer|housenumber|(house|huis)(nr|no)|number/i)
|
52
|
+
ad = f if ( f.to_s =~ /address|street|straat|adres/i)
|
53
|
+
end
|
54
|
+
@params[:hasaddress] = 'unknown'
|
55
|
+
if (ad or hn)
|
56
|
+
if pc
|
57
|
+
@params[:hasaddress] = 'certain'
|
58
|
+
@params[:postcode] = pc
|
59
|
+
elsif pd
|
60
|
+
@params[:hasaddress] = 'maybe'
|
61
|
+
@params[:postcode] = pd
|
62
|
+
end
|
63
|
+
@params[:housenumber] = hn ? hn : ad
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def findUniqueField
|
68
|
+
fields = {}
|
69
|
+
unfield = nil
|
70
|
+
|
71
|
+
return if @content[0][:id]
|
72
|
+
|
73
|
+
@content.each do |h|
|
74
|
+
h[:properties].each do |k,v|
|
75
|
+
fields[k] = {} if fields[k].nil?
|
76
|
+
(fields[k][v] == nil) ? fields[k][v] = 1 : fields[k][v] += 1
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
fields.each_key do |k|
|
81
|
+
if fields[k].length == @params[:rowcount]
|
82
|
+
@params[:unique_id] = unfield = k
|
83
|
+
break
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
if unfield
|
88
|
+
@content.each do |h|
|
89
|
+
h[:id] = h[:properties][unfield]
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
94
|
+
|
95
|
+
def guessName
|
96
|
+
@params[:fields].reverse.each do |k|
|
97
|
+
if(k.to_s =~ RE_A_NAME)
|
98
|
+
@params[:name] = k
|
99
|
+
return
|
100
|
+
end
|
101
|
+
if(k.to_s =~ RE_NAME)
|
102
|
+
@params[:name] = k
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def getFields
|
108
|
+
@params[:fields] = []
|
109
|
+
@content[0][:properties].each_key do |k|
|
110
|
+
@params[:fields] << k
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def guessSRID
|
115
|
+
return if @content[0][:geometry].nil?
|
116
|
+
@params[:srid] = 4326
|
117
|
+
g = @content[0][:geometry][:coordinates]
|
118
|
+
while g[0].is_a?(Array)
|
119
|
+
g = g[0]
|
120
|
+
end
|
121
|
+
lon = g[0]
|
122
|
+
lat = g[1]
|
123
|
+
# if lon > -180.0 and lon < 180.0 and lat > -90.0 and lat < 90.0
|
124
|
+
# @params[:srid] = 4326
|
125
|
+
# els
|
126
|
+
if lon > -7000.0 and lon < 300000.0 and lat > 289000.0 and lat < 629000.0
|
127
|
+
# Dutch new rd system
|
128
|
+
@params[:srid] = 28992
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def findColSep(f)
|
133
|
+
a = f.gets
|
134
|
+
b = f.gets
|
135
|
+
[";","\t","|"].each do |s|
|
136
|
+
return s if (a.split(s).length == b.split(s).length) and b.split(s).length > 1
|
137
|
+
end
|
138
|
+
','
|
139
|
+
end
|
140
|
+
|
141
|
+
def isWkbGeometry(s)
|
142
|
+
begin
|
143
|
+
f = GeoRuby::SimpleFeatures::GeometryFactory::new
|
144
|
+
p = GeoRuby::SimpleFeatures::HexEWKBParser.new(f)
|
145
|
+
p.parse(s)
|
146
|
+
g = f.geometry
|
147
|
+
return g.srid,g.as_json[:type],g
|
148
|
+
rescue Exception=>e
|
149
|
+
end
|
150
|
+
nil
|
151
|
+
end
|
152
|
+
|
153
|
+
def isGeoJSON(s)
|
154
|
+
begin
|
155
|
+
if ['Point', 'MultiPoint', 'LineString', 'MultiLineString', 'Polygon', 'MultiPolygon', 'GeometryCollection'].include?(s[:type])
|
156
|
+
srid = 4326
|
157
|
+
if s[:crs] && s[:crs][:type] == 'OGC'
|
158
|
+
urn = s[:crs][:properties][:urn].split(':')
|
159
|
+
srid = urn.last.to_i if (urn[4] == 'EPSG')
|
160
|
+
end
|
161
|
+
return srid,s[:type],s
|
162
|
+
end
|
163
|
+
rescue Exception=>e
|
164
|
+
end
|
165
|
+
nil
|
166
|
+
end
|
167
|
+
|
168
|
+
|
169
|
+
def findGeometry
|
170
|
+
xfield = nil; xs = true
|
171
|
+
yfield = nil; ys = true
|
172
|
+
@content[0][:properties].each do |k,v|
|
173
|
+
if k.to_s =~ RE_GEO
|
174
|
+
srid,g_type = isWkbGeometry(v)
|
175
|
+
if(srid)
|
176
|
+
@params[:srid] = srid
|
177
|
+
@params[:geomtry_type] = g_type
|
178
|
+
@content.each do |h|
|
179
|
+
a,b,g = isWkbGeometry(h[:properties][k])
|
180
|
+
h[:geometry] = g
|
181
|
+
h[:properties].delete(k)
|
182
|
+
end
|
183
|
+
@params[:hasgeometry] = 'certain'
|
184
|
+
return true
|
185
|
+
end
|
186
|
+
|
187
|
+
srid,g_type = isGeoJSON(v)
|
188
|
+
if(srid)
|
189
|
+
@params[:srid] = srid
|
190
|
+
@params[:geomtry_type] = g_type
|
191
|
+
@content.each do |h|
|
192
|
+
h[:geometry] = h[:properties][k]
|
193
|
+
h[:properties].delete(k)
|
194
|
+
end
|
195
|
+
@params[:hasgeometry] = 'certain'
|
196
|
+
return true
|
197
|
+
end
|
198
|
+
|
199
|
+
@content.each do |h|
|
200
|
+
h[:geometry] = h[:properties][k]
|
201
|
+
h[:properties].delete(k)
|
202
|
+
end
|
203
|
+
@params[:hasgeometry] = 'maybe'
|
204
|
+
return
|
205
|
+
end
|
206
|
+
|
207
|
+
hdc = k.to_sym.downcase
|
208
|
+
if hdc == 'longitude' or hdc == 'lon'
|
209
|
+
xfield=k; xs=false
|
210
|
+
end
|
211
|
+
if hdc == 'latitude' or hdc == 'lat'
|
212
|
+
yfield=k; ys=false
|
213
|
+
end
|
214
|
+
xfield = k if xs and (hdc =~ RE_X)
|
215
|
+
yfield = k if ys and (hdc =~ RE_Y)
|
216
|
+
end
|
217
|
+
|
218
|
+
if xfield and yfield and (xfield != yfield)
|
219
|
+
@params[:hasgeometry] = 'certain'
|
220
|
+
@content.each do |h|
|
221
|
+
h[:geometry] = {:type => 'Point', :coordinates => [h[:properties][xfield].gsub(',','.').to_f, h[:properties][yfield].gsub(',','.').to_f]}
|
222
|
+
h[:properties].delete(yfield)
|
223
|
+
h[:properties].delete(xfield)
|
224
|
+
end
|
225
|
+
@params[:geomtry_type] = 'Point'
|
226
|
+
return true
|
227
|
+
end
|
228
|
+
false
|
229
|
+
end
|
230
|
+
|
231
|
+
|
232
|
+
def readCsv(path)
|
233
|
+
@file = path
|
234
|
+
c=''
|
235
|
+
File.open(path, "r:bom|utf-8") do |fd|
|
236
|
+
c = fd.read
|
237
|
+
end
|
238
|
+
if true != @params[:utf8_fixed]
|
239
|
+
detect = CharlockHolmes::EncodingDetector.detect(c)
|
240
|
+
c = CharlockHolmes::Converter.convert(c, detect[:encoding], 'UTF-8') if detect
|
241
|
+
end
|
242
|
+
c = c.force_encoding('utf-8')
|
243
|
+
@content = []
|
244
|
+
@params[:colsep] = findColSep(StringIO.new(c))
|
245
|
+
csv = CSV.new(c, :col_sep => @params[:colsep], :headers => true, :skip_blanks =>true)
|
246
|
+
csv.each do |row|
|
247
|
+
r = row.to_hash
|
248
|
+
h = {}
|
249
|
+
r.each do |k,v|
|
250
|
+
h[(k.to_sym rescue k) || k] = v
|
251
|
+
end
|
252
|
+
@content << {:properties => h }
|
253
|
+
end
|
254
|
+
findGeometry
|
255
|
+
end
|
256
|
+
|
257
|
+
|
258
|
+
def readJSON(path)
|
259
|
+
@content = []
|
260
|
+
@file = path
|
261
|
+
raw = ''
|
262
|
+
File.open(path, "r:bom|utf-8") do |fd|
|
263
|
+
raw = fd.read
|
264
|
+
end
|
265
|
+
hash = CitySDK::parseJson(raw)
|
266
|
+
|
267
|
+
if hash.is_a?(Hash) and hash[:type] and (hash[:type] == 'FeatureCollection')
|
268
|
+
hash[:features].each do |f|
|
269
|
+
f.delete(:type)
|
270
|
+
@content << f
|
271
|
+
end
|
272
|
+
@params[:hasgeometry] = 'certain'
|
273
|
+
findUniqueField if @content[0][:id].nil?
|
274
|
+
else
|
275
|
+
val,length = nil,0
|
276
|
+
|
277
|
+
if hash.is_a?(Array)
|
278
|
+
val,length = hash,hash.length
|
279
|
+
else
|
280
|
+
hash.each do |k,v|
|
281
|
+
if v.is_a?(Array)
|
282
|
+
val,length = v,v.length if v.length > length
|
283
|
+
end
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
if val
|
288
|
+
val.each do |h|
|
289
|
+
@content << {:properties => h}
|
290
|
+
end
|
291
|
+
end
|
292
|
+
findGeometry
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
|
297
|
+
def sridFromPrj(str)
|
298
|
+
begin
|
299
|
+
connection = Faraday.new :url => "http://prj2epsg.org"
|
300
|
+
resp = connection.get('/search.json', {:mode => 'wkt', :terms => str})
|
301
|
+
if resp.status == 200
|
302
|
+
resp = CitySDK::parseJson resp.body
|
303
|
+
@params[:srid] = resp[:codes][0][:code].to_i
|
304
|
+
end
|
305
|
+
rescue
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
def readShape(path)
|
310
|
+
@content = []
|
311
|
+
@file = path
|
312
|
+
|
313
|
+
prj = path.gsub(/.shp$/i,"") + '.prj'
|
314
|
+
prj = File.exists?(prj) ? File.read(prj) : nil
|
315
|
+
sridFromPrj(prj) if (prj and @params[:srid].nil?)
|
316
|
+
|
317
|
+
@params[:hasgeometry] = 'certain'
|
318
|
+
|
319
|
+
GeoRuby::Shp4r::ShpFile.open(path) do |shp|
|
320
|
+
shp.each do |shape|
|
321
|
+
h = {}
|
322
|
+
h[:geometry] = CitySDK::parseJson(shape.geometry.to_json) #a GeoRuby SimpleFeature
|
323
|
+
h[:properties] = {}
|
324
|
+
att_data = shape.data #a Hash
|
325
|
+
shp.fields.each do |field|
|
326
|
+
h[:properties][field.name.to_sym] = att_data[field.name]
|
327
|
+
end
|
328
|
+
@content << h
|
329
|
+
end
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
def readCsdk(path)
|
334
|
+
h = Marshal.load(File.read(path))
|
335
|
+
@params = h[:config]
|
336
|
+
@content = h[:content]
|
337
|
+
end
|
338
|
+
|
339
|
+
def readZip(path)
|
340
|
+
begin
|
341
|
+
Dir.mktmpdir("cdkfi_#{File.basename(path)}") do |dir|
|
342
|
+
raise "Error unzipping #{path}." if not system "unzip #{path} -d #{dir} > /dev/null 2>&1"
|
343
|
+
Dir.foreach(dir) do |f|
|
344
|
+
next if f =~ /^\./
|
345
|
+
case File.extname(f)
|
346
|
+
when /\.(geo)?json/i
|
347
|
+
readJSON(dir+'/'+f)
|
348
|
+
return
|
349
|
+
when /\.shp/i
|
350
|
+
readShape(dir+'/'+f)
|
351
|
+
return
|
352
|
+
when /\.csv|tsv/i
|
353
|
+
readCsv(dir+'/'+f)
|
354
|
+
return
|
355
|
+
end
|
356
|
+
end
|
357
|
+
end
|
358
|
+
rescue Exception => e
|
359
|
+
raise CitySDK::Exception(e.message, {:originalfile => path}, __FILE__,__LINE__)
|
360
|
+
end
|
361
|
+
end
|
362
|
+
|
363
|
+
def write(path=nil)
|
364
|
+
begin
|
365
|
+
path = @file_path if path.nil?
|
366
|
+
File.open(path+'.csdk',"w") do |fd|
|
367
|
+
fd.write( Marshal.dump({:config=>@params, :content=>@content}) )
|
368
|
+
# fd.write(JSON.pretty_generate({:config=>@params, :content=>@content}))
|
369
|
+
end
|
370
|
+
rescue
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
374
|
+
end
|
375
|
+
|
376
|
+
end
|
377
|
+
|
378
|
+
|
379
|
+
|
380
|
+
# {
|
381
|
+
# :headers => ['aap','noot','titel', 'gid', 'geometry']
|
382
|
+
# :config => {:geom => 'geometry', :name => 'titel', :id => 'gid', :srid => 28892}
|
383
|
+
# :data => [
|
384
|
+
# ['jpo','pipo','naam1','1092',{:type => 'Point', :coordinates => [5.3, 52.4]}],
|
385
|
+
# ['jpa','popi','naam2','1093',{:type => 'Point', :coordinates => [5.1, 52.1]}]
|
386
|
+
# ]
|
387
|
+
# }
|