xdata 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/xdata +41 -28
- data/lib/xdata.rb +1 -1
- data/lib/xdata/file_reader.rb +28 -24
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 36821abb4810857cbd4d1c5f865fbb7f4ee6f040
|
4
|
+
data.tar.gz: be8cc7b12704b7a47771f37ed229a635018ac56c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 731e89b14acb3e84a08220c76f003d3c07f17d21a5a7a3269dd4eb67a4503b59665e8d46053d44afdb97e07a51e075106cb67b899b18c9d4c21592dd917b91f6
|
7
|
+
data.tar.gz: 7c55a23a474789e0c932babc090727f798ccc88f868c333a179deb057126e066ce0ac4d9b3af5d739efb496e10a2daa24a7236c77dee642a8e8009707310b8e1
|
data/bin/xdata
CHANGED
@@ -173,7 +173,7 @@ def saveFileCSV(path=nil)
|
|
173
173
|
|
174
174
|
$file_reader.content.each do |o|
|
175
175
|
a = []
|
176
|
-
o[:properties]
|
176
|
+
o[:properties].each do |k,v|
|
177
177
|
k2 = $file_hash[:alternate_fields][k]
|
178
178
|
a << v if !k2.blank?
|
179
179
|
end
|
@@ -215,7 +215,7 @@ def formatObject(o)
|
|
215
215
|
begin
|
216
216
|
tp = {}
|
217
217
|
# log JSON.pretty_generate($file_hash)
|
218
|
-
o[:properties]
|
218
|
+
o[:properties].each do |k,v|
|
219
219
|
k2 = $file_hash[:alternate_fields][k]
|
220
220
|
tp[k2] = v if !k2.blank?
|
221
221
|
end
|
@@ -248,33 +248,49 @@ end
|
|
248
248
|
|
249
249
|
def fileSummary(vp)
|
250
250
|
|
251
|
-
$outwin
|
251
|
+
if $outwin
|
252
|
+
$outwin.clear
|
252
253
|
|
253
|
-
|
254
|
-
|
254
|
+
$outwin.setpos(vp+=1,0)
|
255
|
+
$outwin.addstr sprintf("%14s #{$file_hash[:file_path]}", "file:")
|
255
256
|
|
256
|
-
$outwin.setpos(vp+=1,0)
|
257
|
-
$outwin.addstr sprintf("%14s #{$file_hash[:rowcount]}", "total rows:")
|
258
|
-
|
259
|
-
$outwin.setpos(vp+=1,0)
|
260
|
-
$outwin.addstr sprintf("%14s #{$file_hash[:unique_id].to_s}","unique id:")
|
261
|
-
|
262
|
-
if $file_hash[:postcode]
|
263
257
|
$outwin.setpos(vp+=1,0)
|
264
|
-
$outwin.addstr sprintf("%14s #{$file_hash[:
|
265
|
-
end
|
258
|
+
$outwin.addstr sprintf("%14s #{$file_hash[:format].to_s}","data format:")
|
266
259
|
|
267
|
-
if $file_hash[:housenumber]
|
268
260
|
$outwin.setpos(vp+=1,0)
|
269
|
-
$outwin.addstr sprintf("%14s #{$file_hash[:
|
270
|
-
end
|
261
|
+
$outwin.addstr sprintf("%14s #{$file_hash[:rowcount]}", "total rows:")
|
271
262
|
|
272
|
-
if $file_hash[:hasgeometry]
|
273
263
|
$outwin.setpos(vp+=1,0)
|
274
|
-
$outwin.addstr sprintf("%14s
|
264
|
+
$outwin.addstr sprintf("%14s #{$file_hash[:unique_id].to_s}","unique id:")
|
265
|
+
|
266
|
+
if $file_hash[:postcode]
|
267
|
+
$outwin.setpos(vp+=1,0)
|
268
|
+
$outwin.addstr sprintf("%14s #{$file_hash[:postcode]}","postcode in:")
|
269
|
+
end
|
270
|
+
|
271
|
+
if $file_hash[:hasgeometry]
|
272
|
+
$outwin.setpos(vp+=1,0)
|
273
|
+
$outwin.addstr sprintf("%14s found in %s; srid: #{$file_hash[:srid]}","geometry:",$file_hash[:hasgeometry])
|
274
|
+
end
|
275
|
+
|
276
|
+
$outwin.refresh
|
277
|
+
else
|
278
|
+
printf("%-17s #{$file_hash[:file_path]}\n", "file:")
|
279
|
+
printf("%-17s #{$file_hash[:format].to_s}\n","data format:")
|
280
|
+
printf("%-17s #{$file_hash[:rowcount]}\n", "total rows:")
|
281
|
+
printf("%-17s #{$file_hash[:unique_id].to_s}\n","unique id:")
|
282
|
+
if $file_hash[:hasgeometry]
|
283
|
+
printf("%-17s found in %s; srid: #{$file_hash[:srid]}\n","geometry:",$file_hash[:hasgeometry])
|
284
|
+
elsif $file_hash[:postcode]
|
285
|
+
printf("%-17s found through %s\n","geometry:",$file_hash[:postcode])
|
286
|
+
end
|
287
|
+
printf("fields:\n")
|
288
|
+
$file_hash[:alternate_fields].each do |k,v|
|
289
|
+
printf("%-14s -> %s\n",k.to_s,v ? v : '<ignored>')
|
290
|
+
end
|
291
|
+
|
275
292
|
end
|
276
293
|
|
277
|
-
$outwin.refresh
|
278
294
|
end
|
279
295
|
|
280
296
|
def loadFile(filePath=nil)
|
@@ -306,7 +322,7 @@ def loadFile(filePath=nil)
|
|
306
322
|
end
|
307
323
|
|
308
324
|
def editFields
|
309
|
-
props = $file_reader.content[rand($file_reader.content.length)][:properties]
|
325
|
+
props = $file_reader.content[rand($file_reader.content.length)][:properties]
|
310
326
|
$outwin.clear
|
311
327
|
outAddstr(1, "For each field, please choose Accept, Rename or Ignore")
|
312
328
|
vp = 3
|
@@ -625,7 +641,7 @@ end
|
|
625
641
|
|
626
642
|
|
627
643
|
def printHelp
|
628
|
-
STDERR.puts "
|
644
|
+
STDERR.puts "\nxdata: Metadata extraction and manipulation tool."
|
629
645
|
STDERR.puts "work in progress"
|
630
646
|
STDERR.puts
|
631
647
|
STDERR.puts "xdata file|url --- load data in interactive mode"
|
@@ -673,11 +689,8 @@ opts.each do |opt, arg|
|
|
673
689
|
end
|
674
690
|
|
675
691
|
FileUtils.mkdir_p(File.expand_path('~') + '/' + ".xdata")
|
676
|
-
|
677
|
-
|
678
692
|
loadFile(ARGV[0]) if (ARGV.length == 1)
|
679
693
|
|
680
|
-
|
681
694
|
if $command.blank?
|
682
695
|
init_screen
|
683
696
|
nl
|
@@ -686,7 +699,8 @@ if $command.blank?
|
|
686
699
|
else
|
687
700
|
case $command[0]
|
688
701
|
when :summary
|
689
|
-
|
702
|
+
fileSummary(0)
|
703
|
+
# puts JSON.pretty_generate($file_hash)
|
690
704
|
exit(0)
|
691
705
|
when :bounds
|
692
706
|
if $file_hash[:bounds]
|
@@ -695,8 +709,7 @@ else
|
|
695
709
|
else
|
696
710
|
STDERR.puts "No geography or postcodes found in file."
|
697
711
|
end
|
698
|
-
else
|
699
|
-
exit(-1)
|
700
712
|
end
|
713
|
+
exit(-1)
|
701
714
|
end
|
702
715
|
|
data/lib/xdata.rb
CHANGED
data/lib/xdata/file_reader.rb
CHANGED
@@ -25,7 +25,7 @@ module XData
|
|
25
25
|
|
26
26
|
RE_Y = /lat|(y.*coord)|(y.*pos.*)|(y.*loc(atie|ation)?)/i
|
27
27
|
RE_X = /lon|lng|(x.*coord)|(x.*pos.*)|(x.*loc(atie|ation)?)/i
|
28
|
-
RE_GEO = /^(geom
|
28
|
+
RE_GEO = /^((geom.*)|location|locatie|coords|coordinates)$/i
|
29
29
|
RE_NAME = /(title|titel|naam|name)/i
|
30
30
|
RE_A_NAME = /^(naam|name|title|titel)$/i
|
31
31
|
|
@@ -135,7 +135,7 @@ module XData
|
|
135
135
|
fields = {}
|
136
136
|
@params[:unique_id] = nil
|
137
137
|
@content.each do |h|
|
138
|
-
h[:properties]
|
138
|
+
h[:properties].each do |k,v|
|
139
139
|
fields[k] = Hash.new(0) if fields[k].nil?
|
140
140
|
fields[k][v] += 1
|
141
141
|
end
|
@@ -167,7 +167,7 @@ module XData
|
|
167
167
|
@params[:fields] = []
|
168
168
|
@params[:alternate_fields] = {}
|
169
169
|
return if @content.blank?
|
170
|
-
@content[0][:properties]
|
170
|
+
@content[0][:properties].each_key do |k|
|
171
171
|
k = (k.to_sym rescue k) || k
|
172
172
|
@params[:fields] << k
|
173
173
|
@params[:alternate_fields][k] = k
|
@@ -279,7 +279,7 @@ module XData
|
|
279
279
|
elsif @params[:postcode]
|
280
280
|
pc = @params[:postcode].to_sym
|
281
281
|
@content.each do |o|
|
282
|
-
p2 = PC4.lookup(o[:properties][
|
282
|
+
p2 = PC4.lookup(o[:properties][pc])
|
283
283
|
if p2
|
284
284
|
geometries << GeoRuby::SimpleFeatures::Point.from_coordinates(p2[0], (@params[:srid] || '4326'))
|
285
285
|
geometries << GeoRuby::SimpleFeatures::Point.from_coordinates(p2[1], (@params[:srid] || '4326'))
|
@@ -299,7 +299,7 @@ module XData
|
|
299
299
|
xs = true
|
300
300
|
ys = true
|
301
301
|
|
302
|
-
@content[0][:properties]
|
302
|
+
@content[0][:properties].each do |k,v|
|
303
303
|
next if k.nil?
|
304
304
|
|
305
305
|
if k.to_s =~ RE_GEO
|
@@ -308,9 +308,9 @@ module XData
|
|
308
308
|
@params[:srid] = srid
|
309
309
|
@params[:geometry_type] = g_type
|
310
310
|
@content.each do |h|
|
311
|
-
a,b,g = is_wkb_geometry?(h[:properties][
|
311
|
+
a,b,g = is_wkb_geometry?(h[:properties][k])
|
312
312
|
h[:geometry] = g
|
313
|
-
h[:properties]
|
313
|
+
h[:properties].delete(k) if delete_column
|
314
314
|
end
|
315
315
|
@params[:hasgeometry] = k
|
316
316
|
return true
|
@@ -321,9 +321,9 @@ module XData
|
|
321
321
|
@params[:srid] = srid
|
322
322
|
@params[:geometry_type] = g_type
|
323
323
|
@content.each do |h|
|
324
|
-
a,b,g = is_wkt_geometry?(h[:properties][
|
324
|
+
a,b,g = is_wkt_geometry?(h[:properties][k])
|
325
325
|
h[:geometry] = g
|
326
|
-
h[:properties]
|
326
|
+
h[:properties].delete(k) if delete_column
|
327
327
|
end
|
328
328
|
@params[:hasgeometry] = k
|
329
329
|
return true
|
@@ -334,7 +334,7 @@ module XData
|
|
334
334
|
@params[:srid] = srid
|
335
335
|
@params[:geometry_type] = g_type
|
336
336
|
@content.each do |h|
|
337
|
-
h[:geometry] = h[:properties][
|
337
|
+
h[:geometry] = h[:properties][k]
|
338
338
|
h[:properties].delete(k) if delete_column
|
339
339
|
end
|
340
340
|
@params[:hasgeometry] = k
|
@@ -358,11 +358,11 @@ module XData
|
|
358
358
|
if xfield and yfield and (xfield != yfield)
|
359
359
|
@params[:hasgeometry] = [xfield,yfield]
|
360
360
|
@content.each do |h|
|
361
|
-
h[:properties][
|
362
|
-
h[:properties][
|
363
|
-
h[:geometry] = {:type => 'Point', :coordinates => [h[:properties][
|
364
|
-
h[:properties]
|
365
|
-
h[:properties]
|
361
|
+
h[:properties][xfield] = h[:properties][xfield] || ''
|
362
|
+
h[:properties][yfield] = h[:properties][yfield] || ''
|
363
|
+
h[:geometry] = {:type => 'Point', :coordinates => [h[:properties][xfield].gsub(',','.').to_f, h[:properties][yfield].gsub(',','.').to_f]}
|
364
|
+
h[:properties].delete(yfield) if delete_column
|
365
|
+
h[:properties].delete(xfield) if delete_column
|
366
366
|
end
|
367
367
|
@params[:geometry_type] = 'Point'
|
368
368
|
@params[:fields].delete(xfield) if @params[:fields] and delete_column
|
@@ -372,8 +372,8 @@ module XData
|
|
372
372
|
# factory = ::RGeo::Cartesian.preferred_factory()
|
373
373
|
@params[:hasgeometry] = [xfield]
|
374
374
|
@content.each do |h|
|
375
|
-
h[:geometry] = geom_from_text(h[:properties][
|
376
|
-
h[:properties]
|
375
|
+
h[:geometry] = geom_from_text(h[:properties][xfield])
|
376
|
+
h[:properties].delete(xfield) if h[:geometry] and delete_column
|
377
377
|
end
|
378
378
|
@params[:geometry_type] = ''
|
379
379
|
@params[:fields].delete(xfield) if @params[:fields] and delete_column
|
@@ -396,6 +396,7 @@ module XData
|
|
396
396
|
c = c.force_encoding('utf-8')
|
397
397
|
c = c.gsub(/\r\n?/, "\n")
|
398
398
|
@content = []
|
399
|
+
@params[:format] = 'CSV'
|
399
400
|
@params[:colsep] = find_col_sep(StringIO.new(c)) unless @params[:colsep]
|
400
401
|
csv = CSV.new(c, :col_sep => @params[:colsep], :headers => true, :skip_blanks =>true)
|
401
402
|
csv.header_convert { |h| h.blank? ? '_' : h.strip.gsub(/\s+/,'_') }
|
@@ -408,7 +409,7 @@ module XData
|
|
408
409
|
r.each do |k,v|
|
409
410
|
h[(k.to_sym rescue k) || k] = v
|
410
411
|
end
|
411
|
-
@content << {properties:
|
412
|
+
@content << {properties: h }
|
412
413
|
index += 1
|
413
414
|
end
|
414
415
|
rescue => e
|
@@ -436,13 +437,13 @@ module XData
|
|
436
437
|
# GeoJSON
|
437
438
|
hash[:features].each do |f|
|
438
439
|
f.delete(:type)
|
439
|
-
f[:properties] = {data: f[:properties]}
|
440
440
|
@content << f
|
441
441
|
end
|
442
|
-
@params[:hasgeometry] = 'GeoJSON'
|
442
|
+
@params[:hasgeometry] = @params[:format] = 'GeoJSON'
|
443
443
|
|
444
444
|
else
|
445
445
|
# Free-form JSON
|
446
|
+
@params[:format] = 'JSON'
|
446
447
|
val,length = nil,0
|
447
448
|
if hash.is_a?(Array)
|
448
449
|
# one big array
|
@@ -458,7 +459,7 @@ module XData
|
|
458
459
|
|
459
460
|
if val
|
460
461
|
val.each do |h|
|
461
|
-
@content << { :properties =>
|
462
|
+
@content << { :properties => h }
|
462
463
|
end
|
463
464
|
end
|
464
465
|
find_geometry
|
@@ -503,6 +504,7 @@ module XData
|
|
503
504
|
|
504
505
|
def read_odata(h)
|
505
506
|
@content = []
|
507
|
+
@params[:format] = "OData"
|
506
508
|
@params[:odata] = {}
|
507
509
|
links = h[:value]
|
508
510
|
links.each do |l|
|
@@ -523,7 +525,7 @@ module XData
|
|
523
525
|
open(odata_json(@params[:odata][:TypedDataSet])) do |f|
|
524
526
|
c = XData::parse_json(f.read)[:value]
|
525
527
|
c.each do |h|
|
526
|
-
@content << { :properties =>
|
528
|
+
@content << { :properties => h }
|
527
529
|
end
|
528
530
|
end
|
529
531
|
|
@@ -544,17 +546,18 @@ module XData
|
|
544
546
|
srid_from_prj(prj) if (prj and @params[:srid].nil?)
|
545
547
|
|
546
548
|
@params[:hasgeometry] = 'ESRI Shape'
|
549
|
+
@params[:format] = "Shape File"
|
547
550
|
|
548
551
|
GeoRuby::Shp4r::ShpFile.open(path) do |shp|
|
549
552
|
shp.each do |shape|
|
550
553
|
h = {}
|
551
554
|
h[:geometry] = XData::parse_json(shape.geometry.to_json) #a GeoRuby SimpleFeature
|
552
|
-
h[:properties] = {
|
555
|
+
h[:properties] = {}
|
553
556
|
att_data = shape.data #a Hash
|
554
557
|
shp.fields.each do |field|
|
555
558
|
s = att_data[field.name]
|
556
559
|
s = s.force_encoding('ISO8859-1') if s.class == String
|
557
|
-
h[:properties][
|
560
|
+
h[:properties][field.name.to_sym] = s
|
558
561
|
end
|
559
562
|
@content << h
|
560
563
|
end
|
@@ -570,6 +573,7 @@ module XData
|
|
570
573
|
begin
|
571
574
|
feed = Feedjira::Feed.parse(data)
|
572
575
|
if feed
|
576
|
+
@params[:format] = "Atom Feed"
|
573
577
|
maxlat = -1000
|
574
578
|
maxlon = -1000
|
575
579
|
minlat = 1000
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: xdata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tom Demeyer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-10-
|
11
|
+
date: 2016-10-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: i18n
|