xdata 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e9d7a71a251d15942157f380276cc36edde1bff
-  data.tar.gz: 5f461ee8d6a15983a5c16f49f885f21ff4e71141
+  metadata.gz: 36821abb4810857cbd4d1c5f865fbb7f4ee6f040
+  data.tar.gz: be8cc7b12704b7a47771f37ed229a635018ac56c
 SHA512:
-  metadata.gz: f631f60e1d264d9560130d6f960af53b7286f449300d614754a9b4619edf7b9eb66e8fe8b3c57f488967f94f04ac701963d41692f1e6850a273afeb26680b731
-  data.tar.gz: bdd3c9ead81f5ca6943fb1d1b8aa4607711374a6f89092c56299f20c8c070dc4a743f070a53ef6bf1573cf5ddc1306702b7dae49e375d53b20d8585d2a7ab9d4
+  metadata.gz: 731e89b14acb3e84a08220c76f003d3c07f17d21a5a7a3269dd4eb67a4503b59665e8d46053d44afdb97e07a51e075106cb67b899b18c9d4c21592dd917b91f6
+  data.tar.gz: 7c55a23a474789e0c932babc090727f798ccc88f868c333a179deb057126e066ce0ac4d9b3af5d739efb496e10a2daa24a7236c77dee642a8e8009707310b8e1

data/bin/xdata

@@ -173,7 +173,7 @@ def saveFileCSV(path=nil)
 
     $file_reader.content.each do |o|
       a = []
-      o[:properties][:data].each do |k,v|
+      o[:properties].each do |k,v|
         k2 = $file_hash[:alternate_fields][k]
         a << v if !k2.blank?
       end
@@ -215,7 +215,7 @@ def formatObject(o)
   begin 
     tp = {}
     # log JSON.pretty_generate($file_hash)
-    o[:properties][:data].each do |k,v|
+    o[:properties].each do |k,v|
       k2 = $file_hash[:alternate_fields][k]
       tp[k2] = v if !k2.blank?
     end
@@ -248,33 +248,49 @@ end
 
 def fileSummary(vp)
   
-  $outwin.clear
+  if $outwin
+    $outwin.clear
 
-  $outwin.setpos(vp+=1,0)
-  $outwin.addstr sprintf("%14s #{$file_hash[:file_path]}", "file:")
+    $outwin.setpos(vp+=1,0)
+    $outwin.addstr sprintf("%14s #{$file_hash[:file_path]}", "file:")
 
-  $outwin.setpos(vp+=1,0)
-  $outwin.addstr sprintf("%14s #{$file_hash[:rowcount]}", "total rows:")
-  
-  $outwin.setpos(vp+=1,0)
-  $outwin.addstr sprintf("%14s #{$file_hash[:unique_id].to_s}","unique id:")
-  
-  if $file_hash[:postcode]
     $outwin.setpos(vp+=1,0)
-    $outwin.addstr sprintf("%14s #{$file_hash[:postcode]}","postcode in:")
-  end
+    $outwin.addstr sprintf("%14s #{$file_hash[:format].to_s}","data format:")
 
-  if $file_hash[:housenumber]
     $outwin.setpos(vp+=1,0)
-    $outwin.addstr sprintf("%14s #{$file_hash[:housenumber]}","address/number in:")
-  end
+    $outwin.addstr sprintf("%14s #{$file_hash[:rowcount]}", "total rows:")
   
-  if $file_hash[:hasgeometry]
     $outwin.setpos(vp+=1,0)
-    $outwin.addstr sprintf("%14s found in %s; srid: #{$file_hash[:srid]}","geometry:",$file_hash[:hasgeometry])
+    $outwin.addstr sprintf("%14s #{$file_hash[:unique_id].to_s}","unique id:")
+  
+    if $file_hash[:postcode]
+      $outwin.setpos(vp+=1,0)
+      $outwin.addstr sprintf("%14s #{$file_hash[:postcode]}","postcode in:")
+    end
+
+    if $file_hash[:hasgeometry]
+      $outwin.setpos(vp+=1,0)
+      $outwin.addstr sprintf("%14s found in %s; srid: #{$file_hash[:srid]}","geometry:",$file_hash[:hasgeometry])
+    end
+  
+    $outwin.refresh
+  else
+    printf("%-17s #{$file_hash[:file_path]}\n", "file:")
+    printf("%-17s #{$file_hash[:format].to_s}\n","data format:")
+    printf("%-17s #{$file_hash[:rowcount]}\n", "total rows:")
+    printf("%-17s #{$file_hash[:unique_id].to_s}\n","unique id:")
+    if $file_hash[:hasgeometry]
+      printf("%-17s found in %s; srid: #{$file_hash[:srid]}\n","geometry:",$file_hash[:hasgeometry])
+    elsif $file_hash[:postcode]
+      printf("%-17s found through %s\n","geometry:",$file_hash[:postcode])
+    end
+    printf("fields:\n")
+    $file_hash[:alternate_fields].each do |k,v|
+      printf("%-14s -> %s\n",k.to_s,v ? v : '<ignored>')
+    end
+
   end
   
-  $outwin.refresh
 end
 
 def loadFile(filePath=nil)
@@ -306,7 +322,7 @@ def loadFile(filePath=nil)
 end
 
 def editFields
-  props = $file_reader.content[rand($file_reader.content.length)][:properties][:data]
+  props = $file_reader.content[rand($file_reader.content.length)][:properties]
   $outwin.clear
   outAddstr(1, "For each field, please choose Accept, Rename or Ignore")
   vp = 3
@@ -625,7 +641,7 @@ end
 
 
 def printHelp
-  STDERR.puts "xdata: Metadata extraction and manipulation tool."
+  STDERR.puts "\nxdata: Metadata extraction and manipulation tool."
   STDERR.puts "work in progress"
   STDERR.puts 
   STDERR.puts "xdata file|url                --- load data in interactive mode"
@@ -673,11 +689,8 @@ opts.each do |opt, arg|
 end
 
 FileUtils.mkdir_p(File.expand_path('~') + '/' + ".xdata")
-
-
 loadFile(ARGV[0]) if (ARGV.length == 1)
 
-
 if $command.blank?
   init_screen
   nl
@@ -686,7 +699,8 @@ if $command.blank?
 else
   case $command[0]
   when :summary
-    puts JSON.pretty_generate($file_hash)
+    fileSummary(0)
+    # puts JSON.pretty_generate($file_hash)
     exit(0)
   when :bounds
     if $file_hash[:bounds]
@@ -695,8 +709,7 @@ else
     else
       STDERR.puts "No geography or postcodes found in file."
     end
-  else
-    exit(-1)
   end
+  exit(-1)
 end
 

data/lib/xdata.rb

@@ -3,5 +3,5 @@ require_relative './xdata/file_reader.rb'
 require_relative './xdata/postcodes_4.rb'
 
 module XData
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
 end

data/lib/xdata/file_reader.rb

@@ -25,7 +25,7 @@ module XData
 
     RE_Y = /lat|(y.*coord)|(y.*pos.*)|(y.*loc(atie|ation)?)/i
     RE_X = /lon|lng|(x.*coord)|(x.*pos.*)|(x.*loc(atie|ation)?)/i
-    RE_GEO = /^(geom(etry)?|location|locatie|coords|coordinates)$/i
+    RE_GEO = /^((geom.*)|location|locatie|coords|coordinates)$/i
     RE_NAME = /(title|titel|naam|name)/i
     RE_A_NAME = /^(naam|name|title|titel)$/i
 
@@ -135,7 +135,7 @@ module XData
       fields = {}
       @params[:unique_id] = nil
       @content.each do |h|
-        h[:properties][:data].each do |k,v|
+        h[:properties].each do |k,v|
           fields[k] = Hash.new(0) if fields[k].nil?
           fields[k][v] += 1
         end
@@ -167,7 +167,7 @@ module XData
       @params[:fields] = []
       @params[:alternate_fields] = {}
       return if @content.blank?
-      @content[0][:properties][:data].each_key do |k|
+      @content[0][:properties].each_key do |k|
         k = (k.to_sym rescue k) || k
         @params[:fields] << k
         @params[:alternate_fields][k] = k
@@ -279,7 +279,7 @@ module XData
       elsif @params[:postcode]
         pc = @params[:postcode].to_sym
         @content.each do |o|
-          p2 = PC4.lookup(o[:properties][:data][pc])
+          p2 = PC4.lookup(o[:properties][pc])
           if p2
             geometries << GeoRuby::SimpleFeatures::Point.from_coordinates(p2[0], (@params[:srid] || '4326'))
             geometries << GeoRuby::SimpleFeatures::Point.from_coordinates(p2[1], (@params[:srid] || '4326'))
@@ -299,7 +299,7 @@ module XData
         xs = true
         ys = true
 
-        @content[0][:properties][:data].each do |k,v|
+        @content[0][:properties].each do |k,v|
           next if k.nil?
 
           if k.to_s =~ RE_GEO
@@ -308,9 +308,9 @@ module XData
               @params[:srid] = srid
               @params[:geometry_type] = g_type
               @content.each do |h|
-                a,b,g = is_wkb_geometry?(h[:properties][:data][k])
+                a,b,g = is_wkb_geometry?(h[:properties][k])
                 h[:geometry] = g
-                h[:properties][:data].delete(k) if delete_column
+                h[:properties].delete(k) if delete_column
               end
               @params[:hasgeometry] = k
               return true
@@ -321,9 +321,9 @@ module XData
               @params[:srid] = srid
               @params[:geometry_type] = g_type
               @content.each do |h|
-                a,b,g = is_wkt_geometry?(h[:properties][:data][k])
+                a,b,g = is_wkt_geometry?(h[:properties][k])
                 h[:geometry] = g
-                h[:properties][:data].delete(k) if delete_column
+                h[:properties].delete(k) if delete_column
               end
               @params[:hasgeometry] = k
               return true
@@ -334,7 +334,7 @@ module XData
               @params[:srid] = srid
               @params[:geometry_type] = g_type
               @content.each do |h|
-                h[:geometry] = h[:properties][:data][k]
+                h[:geometry] = h[:properties][k]
                 h[:properties].delete(k) if delete_column
               end
               @params[:hasgeometry] = k
@@ -358,11 +358,11 @@ module XData
       if xfield and yfield and (xfield != yfield)
         @params[:hasgeometry] = [xfield,yfield]
         @content.each do |h|
-          h[:properties][:data][xfield] = h[:properties][:data][xfield] || ''
-          h[:properties][:data][yfield] = h[:properties][:data][yfield] || ''
-          h[:geometry] = {:type => 'Point', :coordinates => [h[:properties][:data][xfield].gsub(',','.').to_f, h[:properties][:data][yfield].gsub(',','.').to_f]}
-          h[:properties][:data].delete(yfield) if delete_column
-          h[:properties][:data].delete(xfield) if delete_column
+          h[:properties][xfield] = h[:properties][xfield] || ''
+          h[:properties][yfield] = h[:properties][yfield] || ''
+          h[:geometry] = {:type => 'Point', :coordinates => [h[:properties][xfield].gsub(',','.').to_f, h[:properties][yfield].gsub(',','.').to_f]}
+          h[:properties].delete(yfield) if delete_column
+          h[:properties].delete(xfield) if delete_column
         end
         @params[:geometry_type] = 'Point'
         @params[:fields].delete(xfield) if @params[:fields] and delete_column
@@ -372,8 +372,8 @@ module XData
         # factory = ::RGeo::Cartesian.preferred_factory()
         @params[:hasgeometry] = [xfield]
         @content.each do |h|
-          h[:geometry] = geom_from_text(h[:properties][:data][xfield])
-          h[:properties][:data].delete(xfield) if h[:geometry] and delete_column
+          h[:geometry] = geom_from_text(h[:properties][xfield])
+          h[:properties].delete(xfield) if h[:geometry] and delete_column
         end
         @params[:geometry_type] = ''
         @params[:fields].delete(xfield) if @params[:fields] and delete_column
@@ -396,6 +396,7 @@ module XData
       c = c.force_encoding('utf-8')
       c = c.gsub(/\r\n?/, "\n")
       @content = []
+      @params[:format] = 'CSV'
       @params[:colsep] = find_col_sep(StringIO.new(c)) unless @params[:colsep]
       csv = CSV.new(c, :col_sep => @params[:colsep], :headers => true, :skip_blanks =>true)
       csv.header_convert { |h| h.blank? ? '_' : h.strip.gsub(/\s+/,'_')  }
@@ -408,7 +409,7 @@ module XData
           r.each do |k,v|
             h[(k.to_sym rescue k) || k] = v
           end
-          @content << {properties: {data: h} }
+          @content << {properties: h }
           index += 1
         end
       rescue => e
@@ -436,13 +437,13 @@ module XData
         # GeoJSON
         hash[:features].each do |f|
           f.delete(:type)
-          f[:properties] = {data: f[:properties]}
           @content << f
         end
-        @params[:hasgeometry] = 'GeoJSON'
+        @params[:hasgeometry] = @params[:format] = 'GeoJSON'
 
       else
         # Free-form JSON
+        @params[:format] = 'JSON'
         val,length = nil,0
         if hash.is_a?(Array)
            # one big array
@@ -458,7 +459,7 @@ module XData
 
         if val
           val.each do |h|
-            @content << { :properties => {:data => h} }
+            @content << { :properties => h }
           end
         end
         find_geometry
@@ -503,6 +504,7 @@ module XData
 
     def read_odata(h)
       @content = []
+      @params[:format] = "OData"
       @params[:odata] = {}
       links = h[:value]
       links.each do |l|
@@ -523,7 +525,7 @@ module XData
         open(odata_json(@params[:odata][:TypedDataSet])) do |f|
           c = XData::parse_json(f.read)[:value]
           c.each do |h|
-            @content << { :properties => {:data => h} }
+            @content << { :properties => h }
           end
         end
 
@@ -544,17 +546,18 @@ module XData
       srid_from_prj(prj) if (prj and @params[:srid].nil?)
 
       @params[:hasgeometry] = 'ESRI Shape'
+      @params[:format] = "Shape File"
 
       GeoRuby::Shp4r::ShpFile.open(path) do |shp|
         shp.each do |shape|
           h = {}
           h[:geometry] = XData::parse_json(shape.geometry.to_json) #a GeoRuby SimpleFeature
-          h[:properties] = {:data => {}}
+          h[:properties] = {}
           att_data = shape.data #a Hash
           shp.fields.each do |field|
             s = att_data[field.name]
             s = s.force_encoding('ISO8859-1') if s.class == String
-            h[:properties][:data][field.name.to_sym] = s
+            h[:properties][field.name.to_sym] = s
           end
           @content << h
         end
@@ -570,6 +573,7 @@ module XData
       begin 
         feed = Feedjira::Feed.parse(data)
         if feed 
+          @params[:format] = "Atom Feed"
           maxlat = -1000
           maxlon = -1000
           minlat = 1000

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: xdata
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Tom Demeyer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-11 00:00:00.000000000 Z
+date: 2016-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: i18n