citysdk 0.1.2.5 → 1.0

data/lib/citysdk/file_reader.rb

@@ -7,12 +7,11 @@ require 'tmpdir'
 
 module CitySDK
 
-
   class FileReader
     
-    RE_Y = /lat|(y.*coord)|(y.*loc(atie|ation)?)/i
-    RE_X = /lon|lng|(x.*coord)|(x.*loc(atie|ation)?)/i
-    RE_GEO = /^geom(etry)?|location|locatie$/i
+    RE_Y = /lat|(y.*coord)|(y.*pos.*)|(y.*loc(atie|ation)?)/i
+    RE_X = /lon|lng|(x.*coord)|(x.*pos.*)|(x.*loc(atie|ation)?)/i
+    RE_GEO = /^(geom(etry)?|location|locatie|coords|coordinates)$/i
     RE_NAME = /(title|titel|naam|name)/i
     RE_A_NAME = /^(naam|name|title|titel)$/i
     
@@ -41,68 +40,74 @@ module CitySDK
         end
       end
       
-
       @params[:rowcount] = @content.length
       getFields unless @params[:fields]
       guessName unless @params[:name]
       guessSRID unless @params[:srid]
       findUniqueField  unless @params[:unique_id]
       getAddress unless @params[:hasaddress]
-      @params[:hasgeometry] = 'unknown' if @params[:hasgeometry].nil?
+      setId_Name
     end
     
     def getAddress
       pd = pc = hn = ad = false
+      @params[:housenumber] = nil
+      @params[:hasaddress] = 'unknown'
+      @params[:postcode] = nil
       @params[:fields].reverse.each do |f|
-        pd = f if ( f.to_s =~ /postcode|post/i )
         pc = f if ( f.to_s =~ /^(post|zip|postal)code$/i )
         hn = f if ( f.to_s =~ /huisnummer|housenumber|(house|huis)(nr|no)|number/i)
         ad = f if ( f.to_s =~ /address|street|straat|adres/i)
       end
-      @params[:hasaddress] = 'unknown'
-      if (ad or hn)
-        if pc 
-          @params[:hasaddress] = 'certain'
-          @params[:postcode] = pc
-        elsif pd
-          @params[:hasaddress] = 'maybe'
-          @params[:postcode] = pd
-        end
-        @params[:housenumber] = hn ? hn : ad
+      if pc and (ad or hn)
+        @params[:hasaddress] = 'certain'
       end
-    end
-
+      @params[:postcode] = pc
+      @params[:housenumber] = hn ? hn : ad
 
+    end
 
+    def setId_Name
+      count = 123456
+      if @params[:unique_id]
+        @content.each do |h|
+          h[:properties][:id] = h[:properties][:data][@params[:unique_id]]
+          h[:properties][:title] = h[:properties][:data][@params[:name]] if @params[:name]
+        end
+      else
+        @params[:unique_id] = :csdk_gen
+        # @params[:fields] << :csdk_gen
+        # @params[:original_fields] << :csdk_gen
+        # @params[:alternate_fields][:csdk_gen] = :csdk_gen
+        @content.each do |h|
+          h[:properties][:id] = "cg_#{count}"
+          h[:properties][:title] = h[:properties][:data][@params[:name]] if @params[:name]
+          count += 1
+        end
+      end
+    end
+    
     def findUniqueField
       fields = {}
-      unfield = nil
-
-      return if @content[0][:id]
-
+      @params[:unique_id] = nil
       @content.each do |h|
-        h[:properties].each do |k,v|
-          fields[k] = {} if fields[k].nil?
-          (fields[k][v] == nil) ? fields[k][v] = 1 : fields[k][v] += 1
+        h[:properties][:data].each do |k,v|
+          fields[k] = Hash.new(0) if fields[k].nil?
+          fields[k][v] += 1
         end
       end
 
       fields.each_key do |k|
         if fields[k].length == @params[:rowcount]
-          @params[:unique_id] = unfield = k
+          @params[:unique_id] = k
           break
         end
       end
-
-      if unfield
-        @content.each do |h|
-          h[:id] = h[:properties][unfield]
-        end
-      end
-
+      
     end
 
     def guessName
+      @params[:name] = nil
       @params[:fields].reverse.each do |k|
         if(k.to_s =~ RE_A_NAME)
           @params[:name] = k
@@ -116,13 +121,18 @@ module CitySDK
 
     def getFields
       @params[:fields] = []
-      @content[0][:properties].each_key do |k|
-        @params[:fields] << (k.to_sym rescue k) || k
+      @params[:original_fields] = []
+      @params[:alternate_fields] = {}
+      @content[0][:properties][:data].each_key do |k|
+        k = (k.to_sym rescue k) || k
+        @params[:fields] << k
+        @params[:original_fields] << k
+        @params[:alternate_fields][k] = k
       end
     end
     
     def guessSRID
-      return if @content[0][:geometry].nil?
+      return unless @content[0][:geometry] and @content[0][:geometry].class == Hash
       @params[:srid] = 4326 
       g = @content[0][:geometry][:coordinates]
       if(g)
@@ -133,11 +143,13 @@ module CitySDK
         lat = g[1]
         # if lon > -180.0 and lon < 180.0 and lat > -90.0 and lat < 90.0 
         #   @params[:srid] = 4326 
-        # els
-        if lon > -7000.0 and lon < 300000.0 and lat > 289000.0 and lat < 629000.0
+        # else
+        if lon.between?(-7000.0,300000.0) and lat.between?(289000.0,629000.0)
           # Dutch new rd system
           @params[:srid] = 28992
         end
+      else 
+        
       end
     end
     
@@ -157,18 +169,36 @@ module CitySDK
         p.parse(s)
         g = f.geometry
         return g.srid,g.as_json[:type],g
-      rescue Exception=>e
+      rescue => e
+      end
+      nil
+    end
+
+    def isWktGeometry(s)
+      begin
+        f = GeoRuby::SimpleFeatures::GeometryFactory::new
+        p = GeoRuby::SimpleFeatures::EWKTParser.new(f)
+        p.parse(s)
+        g = f.geometry
+        return g.srid,g.as_json[:type],g
+      rescue => e
       end
       nil
     end
 
+    GEOMETRIES = ["point", "multipoint", "linestring", "multilinestring", "polygon", "multipolygon"]
     def isGeoJSON(s)
+      return nil if s.class != Hash
       begin
-        if ['Point', 'MultiPoint', 'LineString', 'MultiLineString', 'Polygon', 'MultiPolygon', 'GeometryCollection'].include?(s[:type])
+        if GEOMETRIES.include?(s[:type].downcase)
           srid = 4326
-          if s[:crs] && s[:crs][:type] == 'OGC'
-            urn = s[:crs][:properties][:urn].split(':')
-            srid = urn.last.to_i if (urn[4] == 'EPSG')
+          if s[:crs] and s[:crs][:properties]
+            if s[:crs][:type] == 'OGC'
+              urn = s[:crs][:properties][:urn].split(':')
+              srid = urn.last.to_i if (urn[4] == 'EPSG')
+            elsif s[:crs][:type] == 'EPSG'
+              srid = s[:crs][:properties][:code]
+            end
           end
           return srid,s[:type],s
         end
@@ -176,98 +206,151 @@ module CitySDK
       end
       nil
     end
+
+    def geomFromText(coords)
+      # begin 
+      #   a = factory.parse_wkt(coords)
+      # rescue
+      # end
+
+      if coords =~ /^(\w+)(.+)/
+        if GEOMETRIES.include?($1.downcase)
+          type = $1.capitalize
+          coor = $2.gsub('(','[').gsub(')',']')
+          coor = coor.gsub(/([-+]?[0-9]*\.?[0-9]+)\s+([-+]?[0-9]*\.?[0-9]+)/) { "[#{$1},#{$2}]" }
+          coor = JSON.parse(coor)
+          return { :type => type, 
+            :coordinates => coor }
+        end
+      end
+      {}
+    end
     
-    
-    def findGeometry
-      xfield = nil; xs = true
-      yfield = nil; ys = true
-      @content[0][:properties].each do |k,v|
-        next if k.nil?
-        if k.to_s =~ RE_GEO
-          srid,g_type = isWkbGeometry(v)
-          if(srid)
-            @params[:srid] = srid
-            @params[:geomtry_type] = g_type
-            @content.each do |h|
-              a,b,g = isWkbGeometry(h[:properties][k])
-              h[:geometry] = g
-              h[:properties].delete(k)
+    def findGeometry(xfield=nil, yfield=nil)
+      unless(xfield and yfield)
+        @params[:hasgeometry] = nil
+        xs = true
+        ys = true
+
+        @content[0][:properties][:data].each do |k,v|
+          next if k.nil?
+
+          if k.to_s =~ RE_GEO
+            
+            srid,g_type = isWkbGeometry(v)
+            if(srid)
+              @params[:srid] = srid
+              @params[:geomtry_type] = g_type
+              @content.each do |h|
+                a,b,g = isWkbGeometry(h[:properties][:data][k])
+                h[:geometry] = g
+                h[:properties][:data].delete(k)
+              end
+              @params[:hasgeometry] = k
+              return true
             end
-            @params[:hasgeometry] = 'certain'
-            return true
-          end
-          
-          srid,g_type = isGeoJSON(v)
-          if(srid)
-            @params[:srid] = srid
-            @params[:geomtry_type] = g_type
-            @content.each do |h|
-              h[:geometry] = h[:properties][k]
-              h[:properties].delete(k)
+            
+            
+            srid,g_type = isWktGeometry(v)
+            if(srid)
+              @params[:srid] = srid
+              @params[:geomtry_type] = g_type
+              @content.each do |h|
+                a,b,g = isWktGeometry(h[:properties][:data][k])
+                h[:geometry] = g
+                h[:properties][:data].delete(k)
+              end
+              @params[:hasgeometry] = k
+              return true
             end
-            @params[:hasgeometry] = 'certain'
-            return true
+            
+            srid,g_type = isGeoJSON(v)
+            if(srid)
+              @params[:srid] = srid
+              @params[:geomtry_type] = g_type
+              @content.each do |h|
+                h[:geometry] = h[:properties][:data][k]
+                h[:properties].delete(k)
+              end
+              @params[:hasgeometry] = k
+              return true
+            end
+            
           end
           
-          @content.each do |h|
-            h[:geometry] = h[:properties][k]
-            h[:properties].delete(k)
+          hdc = k.to_s.downcase
+          if hdc == 'longitude' or hdc == 'lon' or hdc == 'x'
+            xfield=k; xs=false
           end
-          @params[:hasgeometry] = 'maybe'
-          return
-        end
-        
-        hdc = k.to_sym.downcase
-        if hdc == 'longitude' or hdc == 'lon'
-          xfield=k; xs=false
-        end
-        if hdc == 'latitude' or hdc == 'lat'
-          yfield=k; ys=false
+          if hdc == 'latitude' or hdc == 'lat' or hdc == 'y'
+            yfield=k; ys=false
+          end
+          xfield = k if xs and (hdc =~ RE_X) 
+          yfield = k if ys and (hdc =~ RE_Y)
         end
-        xfield = k if xs and (hdc =~ RE_X) 
-        yfield = k if ys and (hdc =~ RE_Y)
       end
 
       if xfield and yfield and (xfield != yfield)
-        @params[:hasgeometry] = 'certain'
+        @params[:hasgeometry] = [xfield,yfield].to_s
         @content.each do |h|
-          h[:geometry] = {:type => 'Point', :coordinates => [h[:properties][xfield].gsub(',','.').to_f, h[:properties][yfield].gsub(',','.').to_f]}
-          h[:properties].delete(yfield)
-          h[:properties].delete(xfield)
+          h[:geometry] = {:type => 'Point', :coordinates => [h[:properties][:data][xfield].gsub(',','.').to_f, h[:properties][:data][yfield].gsub(',','.').to_f]}
+          h[:properties][:data].delete(yfield)
+          h[:properties][:data].delete(xfield)
         end
         @params[:geomtry_type] = 'Point'
+        @params[:fields].delete(xfield) if @params[:fields]
+        @params[:fields].delete(yfield) if @params[:fields]
+        return true
+      elsif (xfield and yfield)
+        # factory = ::RGeo::Cartesian.preferred_factory()
+        @params[:hasgeometry] = "[#{xfield}]"
+        @content.each do |h|
+          h[:geometry] = geomFromText(h[:properties][:data][xfield])
+          h[:properties][:data].delete(xfield) if h[:geometry]
+        end
+        @params[:geomtry_type] = ''
+        @params[:fields].delete(xfield) if @params[:fields]
         return true
       end
+      
+      
       false
     end
     
-    
     def readCsv(path)
       @file = path
       c=''
       File.open(path, "r:bom|utf-8") do |fd|
         c = fd.read
       end
-      if true != @params[:utf8_fixed]
+      unless @params[:utf8_fixed]
         detect = CharlockHolmes::EncodingDetector.detect(c)
         c =	CharlockHolmes::Converter.convert(c, detect[:encoding], 'UTF-8') if detect
       end
       c = c.force_encoding('utf-8')
+      c = c.gsub(/\r\n?/, "\n")
       @content = []
-      @params[:colsep] = findColSep(StringIO.new(c))
+      @params[:colsep] = findColSep(StringIO.new(c)) unless @params[:colsep]
       csv = CSV.new(c, :col_sep => @params[:colsep], :headers => true, :skip_blanks =>true)
-      csv.each do |row|
-        r = row.to_hash
-        h = {}
-        r.each do |k,v|
-          h[(k.to_sym rescue k) || k] = v
+      csv.header_convert { |h| h.blank? ? '_' : h.strip.gsub(/\s+/,'_')  }
+      csv.convert { |h| h ? h.strip : '' }
+      index = 0
+      begin 
+        csv.each do |row|
+          r = row.to_hash
+          h = {}
+          r.each do |k,v|
+            h[(k.to_sym rescue k) || k] = v
+          end
+          @content << {properties: {data: h} }
+          index += 1
         end
-        @content << {:properties => h }
+      rescue => e
+        raise CitySDK::Exception.new("Read CSV; line #{index}; #{e.message}")
       end
       findGeometry
     end
 
-
     def readJSON(path)
       @content = []
       @file = path
@@ -278,20 +361,23 @@ module CitySDK
       hash = CitySDK::parseJson(raw)
 
       if hash.is_a?(Hash) and hash[:type] and (hash[:type] == 'FeatureCollection')
+        # GeoJSON
         hash[:features].each do |f|
           f.delete(:type)
+          f[:properties] = {data: f[:properties]}
           @content << f
         end
-        @params[:hasgeometry] = 'certain'
-        findUniqueField if @content[0][:id].nil?
+        @params[:hasgeometry] = 'GeoJSON'
       else
+        # Free-form JSON
         val,length = nil,0
-
         if hash.is_a?(Array)
+           # one big array
            val,length = hash,hash.length
         else
           hash.each do |k,v|
             if v.is_a?(Array)
+              # the longest array value in the Object
               val,length = v,v.length if v.length > length
             end
           end
@@ -299,19 +385,18 @@ module CitySDK
         
         if val
           val.each do |h|
-            @content << {:properties => h}
+            @content << { :properties => {:data => h} }
           end
         end
         findGeometry
       end
     end
 
-
     def sridFromPrj(str)
       begin
         connection = Faraday.new :url => "http://prj2epsg.org"
         resp = connection.get('/search.json', {:mode => 'wkt', :terms => str})
-        if resp.status == 200 
+        if resp.status.between?(200, 299) 
           resp = CitySDK::parseJson resp.body
           @params[:srid] = resp[:codes][0][:code].to_i
         end
@@ -328,19 +413,19 @@ module CitySDK
       prj = File.exists?(prj) ? File.read(prj) : nil
       sridFromPrj(prj) if (prj and @params[:srid].nil?)
       
-      @params[:hasgeometry] = 'certain'
+      @params[:hasgeometry] = 'ESRI Shape'
       
       
       GeoRuby::Shp4r::ShpFile.open(path) do |shp|
         shp.each do |shape|
           h = {}
           h[:geometry] = CitySDK::parseJson(shape.geometry.to_json) #a GeoRuby SimpleFeature
-          h[:properties] = {}
+          h[:properties] = {:data => {}}
           att_data = shape.data #a Hash
           shp.fields.each do |field|
             s = att_data[field.name]
             s = s.force_encoding('ISO8859-1') if s.class == String
-            h[:properties][field.name.to_sym] = s
+            h[:properties][:data][field.name.to_sym] = s
           end
           @content << h
         end
@@ -398,13 +483,3 @@ module CitySDK
   
 end
 
-
-
-# {
-#   :headers => ['aap','noot','titel', 'gid', 'geometry']
-#   :config => {:geom => 'geometry', :name => 'titel', :id => 'gid', :srid => 28892}
-#   :data => [
-#     ['jpo','pipo','naam1','1092',{:type => 'Point', :coordinates => [5.3, 52.4]}],
-#     ['jpa','popi','naam2','1093',{:type => 'Point', :coordinates => [5.1, 52.1]}]
-#   ]
-# }