cartodb-importer 0.2.14 → 0.2.15

data/cartodb-importer.gemspec

@@ -6,7 +6,7 @@ Gem::Specification.new do |s|
   s.name        = "cartodb-importer"
   s.version     = CartoDB::Importer::VERSION
   s.platform    = Gem::Platform::RUBY
-  s.authors     = ["Fernando Blat", "Andrew Hill", "Javier de la Torre"]
+  s.authors     = ["Fernando Blat", "Andrew Hill", "Javier de la Torre", "Simon Tokumine"]
   s.email       = ["andrew@vizzuality.com"]
   s.homepage    = ""
   s.summary     = %q{Import CSV, SHP, and other files with data into a PostgreSQL table}
@@ -20,14 +20,16 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   
   s.add_runtime_dependency "pg", "~> 0.11"
-  s.add_runtime_dependency "sequel"
-  s.add_runtime_dependency "roo"
-  s.add_runtime_dependency "spreadsheet"
-  s.add_runtime_dependency "google-spreadsheet-ruby"
-  s.add_runtime_dependency "rubyzip"
-  s.add_runtime_dependency "builder"
+  s.add_runtime_dependency "sequel", "~> 3.28.0"
+  s.add_runtime_dependency "roo", "~> 1.9.7"
+  s.add_runtime_dependency "spreadsheet", "~> 0.6.5.9"
+  s.add_runtime_dependency "google-spreadsheet-ruby", "~> 0.1.5"
+  s.add_runtime_dependency "rubyzip", "~> 0.9.4"
+  s.add_runtime_dependency "builder", "~> 3.0.0"
+  s.add_runtime_dependency "rgeo", "~> 0.3.2"
+  s.add_runtime_dependency "rgeo-geojson", "~> 0.2.1"
   
-  s.add_development_dependency "rspec"
-  s.add_development_dependency "mocha"
-  s.add_development_dependency "ruby-debug19"
+  s.add_development_dependency "rspec", "~> 2.6.0"
+  s.add_development_dependency "mocha", "~> 0.10.0"
+  s.add_development_dependency "ruby-debug19", "~> 0.11.6"
 end

data/lib/cartodb-exporter.rb

@@ -4,6 +4,8 @@ require 'rubygems'
 require 'bundler'
 Bundler.setup
 
+require 'rgeo'
+require 'rgeo/geo_json'
 require 'roo'
 require 'csv'
 require 'tempfile'

data/lib/cartodb-importer.rb

@@ -4,6 +4,8 @@ require 'rubygems'
 require 'bundler'
 Bundler.setup
 
+require 'rgeo'
+require 'rgeo/geo_json'
 require 'roo'
 require 'csv'
 require 'tempfile'

data/lib/cartodb-importer/importer.rb

@@ -175,7 +175,7 @@ module CartoDB
         
         @table_created = true
         
-        FileUtils.rm_rf(path)
+        FileUtils.rm_rf(Dir.glob(path))
         rows_imported = @db_connection["SELECT count(*) as count from #{@suggested_name}"].first[:count]
         
         return OpenStruct.new({
@@ -209,39 +209,82 @@ module CartoDB
           end
         end
         
-        #Now, if there is a ltitude and longitude column, lets create a the_geom for it
-        latitude_possible_names = "'latitude','lat','latitudedecimal','latitud','lati'"
-        longitude_possible_names = "'longitude','lon','lng','longitudedecimal','longitud','long'"
-        
-        matching_latitude = nil
-        res = @db_connection["select column_name from information_schema.columns where table_name ='#{@suggested_name}' 
-          and lower(column_name) in (#{latitude_possible_names}) LIMIT 1"]
-        if !res.first.nil?
-          matching_latitude= res.first[:column_name]
-        end
-        matching_longitude = nil
-        res = @db_connection["select column_name from information_schema.columns where table_name ='#{@suggested_name}' 
-          and lower(column_name) in (#{longitude_possible_names}) LIMIT 1"]
-        if !res.first.nil?
-          matching_longitude= res.first[:column_name]
-        end        
-        
-        
-        if matching_latitude and matching_longitude
-            #we know there is a latitude/longitude columns
-            @db_connection.run("SELECT AddGeometryColumn('#{@suggested_name}','the_geom',4326, 'POINT', 2);")
-            @db_connection.run("UPDATE \"#{@suggested_name}\" SET the_geom = ST_GeomFromText('POINT('|| \"#{matching_longitude}\" ||' '|| \"#{matching_latitude}\" ||')',4326)
-              WHERE \"#{matching_longitude}\" IS NOT NULL AND \"#{matching_latitude}\" IS NOT NULL AND \"#{matching_longitude}\"<>'' AND \"#{matching_latitude}\"<>''")
-            @db_connection.run("CREATE INDEX \"#{@suggested_name}_the_geom_gist\" ON \"#{@suggested_name}\" USING GIST (the_geom)")
+        # Importing CartoDB CSV exports
+        # ===============================
+        # * if there is a column already called the_geom
+        # * if there is geojson in it
+        # * rename column to the_geom_orig
+        # * create a new column with the correct type (Assume 4326) "the_geom_temp"
+        # * loop over table and parse geojson into postgis geometries
+        # * drop the_geom_orig
+        #
+        # TODO: move the geom over using ST_FromGeoJSON once inside PostGIS 2.0
+        if column_names.include? "the_geom"        
+          if res = @db_connection["select the_geom from #{@suggested_name} limit 1"].first
+            
+            # attempt to read as geojson. If it fails, continue
+            begin
+              geojson       = RGeo::GeoJSON.decode(res[:the_geom], :json_parser => :json)
+              geometry_type = geojson.geometry_type.type_name.upcase           
+
+              if geometry_type
+                # move original geometry column around
+                @db_connection.run("ALTER TABLE #{@suggested_name} RENAME COLUMN the_geom TO the_geom_orig;")                
+                @db_connection.run("SELECT AddGeometryColumn('#{@suggested_name}','the_geom',4326, '#{geometry_type}', 2)")
+                @db_connection.run("CREATE INDEX #{@suggested_name}_the_geom_gist ON #{@suggested_name} USING GIST (the_geom)")
+                                
+                # loop through old geom parsing into the_geom. 
+                # TODO: Should probably window this
+                @db_connection["select the_geom_orig from #{@suggested_name}"].each do |res|
+                  begin
+                    geojson = RGeo::GeoJSON.decode(res[:the_geom_orig], :json_parser => :json)
+                    @db_connection.run("UPDATE #{@suggested_name} SET the_geom = ST_GeomFromText('#{geojson.as_text}', 4326) WHERE the_geom_orig = '#{res[:the_geom_orig]}'")                       
+                  rescue => e
+                    runlog.err << "silently fail conversion #{geojson.inspect} to #{@suggested_name}. #{e.inspect}"
+                  end
+                end  
+                
+                # Drop original the_geom column
+                @db_connection.run("ALTER TABLE #{@suggested_name} DROP COLUMN the_geom_orig")
+              end                  
+            rescue => e
+              runlog.err << "failed to read geojson for #{@suggested_name}. #{e.inspect}"
+            end
+          end
         end
         
-        
-        
-        
+        # if there is no the_geom, and there are latitude and longitude columns, create the_geom
+        unless column_names.include? "the_geom"
+          
+          latitude_possible_names = "'latitude','lat','latitudedecimal','latitud','lati'"
+          longitude_possible_names = "'longitude','lon','lng','longitudedecimal','longitud','long'"
+        
+          matching_latitude = nil
+          res = @db_connection["select column_name from information_schema.columns where table_name ='#{@suggested_name}' 
+            and lower(column_name) in (#{latitude_possible_names}) LIMIT 1"]
+          if !res.first.nil?
+            matching_latitude= res.first[:column_name]
+          end
+          matching_longitude = nil
+          res = @db_connection["select column_name from information_schema.columns where table_name ='#{@suggested_name}' 
+            and lower(column_name) in (#{longitude_possible_names}) LIMIT 1"]
+          if !res.first.nil?
+            matching_longitude= res.first[:column_name]
+          end        
+        
+        
+          if matching_latitude and matching_longitude
+              #we know there is a latitude/longitude columns
+              @db_connection.run("SELECT AddGeometryColumn('#{@suggested_name}','the_geom',4326, 'POINT', 2);")
+              @db_connection.run("UPDATE \"#{@suggested_name}\" SET the_geom = ST_GeomFromText('POINT('|| \"#{matching_longitude}\" ||' '|| \"#{matching_latitude}\" ||')',4326)
+                WHERE \"#{matching_longitude}\" IS NOT NULL AND \"#{matching_latitude}\" IS NOT NULL AND \"#{matching_longitude}\"<>'' AND \"#{matching_latitude}\"<>''")
+              @db_connection.run("CREATE INDEX \"#{@suggested_name}_the_geom_gist\" ON \"#{@suggested_name}\" USING GIST (the_geom)")
+          end
+        end
         
         @table_created = true
         
-        FileUtils.rm_rf(path)
+        FileUtils.rm_rf(Dir.glob(path))
         rows_imported = @db_connection["SELECT count(*) as count from #{@suggested_name}"].first[:count]
         
         return OpenStruct.new({

data/lib/cartodb-importer/version.rb

@@ -1,6 +1,6 @@
 module CartoDB
   class Importer
-    VERSION = "0.2.14"
+    VERSION = "0.2.15"
   end
 end
     

data/spec/import_spec.rb

@@ -254,7 +254,7 @@ describe CartoDB::Importer do
       columns = db_connection.schema(:vizzuality).map{|s| s[0].to_s}
       
       expected_columns = ["gid", "subclass", "x", "y", "length", "area", "angle", "name", 
-        "pid", "lot_navteq", "version_na", "vitesse_sp", "id", "nombrerest", "tipocomida", "the_geom"]
+        "pid", "lot_navteq", "version_na", "vitesse_sp", "id", "nombrerest", "tipocomida"]
       (columns & expected_columns).sort.should == expected_columns.sort
     end
     
@@ -346,7 +346,7 @@ describe CartoDB::Importer do
   end  
   
   describe "Import CSV with latidude/logitude" do
-    it "should import estaciones2.csv" do
+    it "should import walmart.csv" do
       importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/walmart.csv", __FILE__),
                                    :database => "cartodb_importer_test", :username => 'postgres', :password => '',
                                    :host => 'localhost', :port => 5432, :suggested_name => 'walmart'
@@ -358,7 +358,7 @@ describe CartoDB::Importer do
   end
   
   describe "Import CSV with lat/lon" do
-    it "should import estaciones2.csv" do
+    it "should import walmart.csv" do
       importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/walmart_latlon.csv", __FILE__),
                                    :database => "cartodb_importer_test", :username => 'postgres', :password => '',
                                    :host => 'localhost', :port => 5432, :suggested_name => 'walmart_latlon'
@@ -368,6 +368,52 @@ describe CartoDB::Importer do
       result.import_type.should == '.csv'
     end
   end  
+
+  describe "Import CartoDB CSV export with lat/lon" do
+    it "should import CartoDB_csv_export.zip" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/CartoDB_csv_export.zip", __FILE__),
+                                   :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                   :host => 'localhost', :port => 5432, :suggested_name => 'cartodb_csv_export'
+      result = importer.import!
+      result.name.should == 'cartodb_csv_export'
+      result.rows_imported.should == 155
+      result.import_type.should == '.csv'
+    end
+  end  
+
+  # TODO: check that the_geom is now a real geometry built from geojson.
+  describe "Import CartoDB CSV export with the_geom in geojson" do
+    it "should import CartoDB_csv_multipoly_export.zip" do
+      opt = {:import_from_file => File.expand_path("../support/data/CartoDB_csv_multipoly_export.zip", __FILE__),
+                 :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                 :host => 'localhost', :port => 5432, :suggested_name => 'cartodb_csv_multipoly_export'}
+      importer = CartoDB::Importer.new opt
+      result = importer.import!
+      result.name.should == 'cartodb_csv_multipoly_export'
+      result.rows_imported.should == 601
+      result.import_type.should == '.csv'
+      
+      # test geometry returned is legit
+      pg = "postgres://#{opt[:username]}:#{opt[:password]}@#{opt[:host]}:#{opt[:port]}/#{opt[:database]}" 
+      sql = "select ST_AsGeoJSON(the_geom,0) as geom from cartodb_csv_multipoly_export limit 1"
+      db_connection = Sequel.connect(pg)            
+      res = db_connection[sql].first[:geom]
+      res.should == '{"type":"MultiPolygon","coordinates":[[[[2,39],[2,39],[2,39],[2,39],[2,39]]]]}'
+    end
+  end  
+
+  describe "Import CartoDB SHP export with lat/lon" do
+    it "should import CartoDB_shp_export.zip" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/CartoDB_shp_export.zip", __FILE__),
+                                   :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                   :host => 'localhost', :port => 5432, :suggested_name => 'cartodb_shp_export'
+      result = importer.import!
+      result.name.should == 'cartodb_shp_export'
+      result.rows_imported.should == 155
+      result.import_type.should == '.shp'
+    end
+  end  
+
   
   
 end

metadata

@@ -1,21 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: cartodb-importer
 version: !ruby/object:Gem::Version
-  version: 0.2.14
+  version: 0.2.15
   prerelease: 
 platform: ruby
 authors:
 - Fernando Blat
 - Andrew Hill
 - Javier de la Torre
+- Simon Tokumine
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-03 00:00:00.000000000Z
+date: 2011-10-04 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
-  requirement: &70348306305600 !ruby/object:Gem::Requirement
+  requirement: &70233688939180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -23,106 +24,128 @@ dependencies:
         version: '0.11'
   type: :runtime
   prerelease: false
-  version_requirements: *70348306305600
+  version_requirements: *70233688939180
 - !ruby/object:Gem::Dependency
   name: sequel
-  requirement: &70348306288700 !ruby/object:Gem::Requirement
+  requirement: &70233688938600 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.28.0
   type: :runtime
   prerelease: false
-  version_requirements: *70348306288700
+  version_requirements: *70233688938600
 - !ruby/object:Gem::Dependency
   name: roo
-  requirement: &70348306287760 !ruby/object:Gem::Requirement
+  requirement: &70233688938100 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.9.7
   type: :runtime
   prerelease: false
-  version_requirements: *70348306287760
+  version_requirements: *70233688938100
 - !ruby/object:Gem::Dependency
   name: spreadsheet
-  requirement: &70348306287120 !ruby/object:Gem::Requirement
+  requirement: &70233688937600 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.5.9
   type: :runtime
   prerelease: false
-  version_requirements: *70348306287120
+  version_requirements: *70233688937600
 - !ruby/object:Gem::Dependency
   name: google-spreadsheet-ruby
-  requirement: &70348306286400 !ruby/object:Gem::Requirement
+  requirement: &70233688937120 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.5
   type: :runtime
   prerelease: false
-  version_requirements: *70348306286400
+  version_requirements: *70233688937120
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &70348306285500 !ruby/object:Gem::Requirement
+  requirement: &70233688936400 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.4
   type: :runtime
   prerelease: false
-  version_requirements: *70348306285500
+  version_requirements: *70233688936400
 - !ruby/object:Gem::Dependency
   name: builder
-  requirement: &70348306284860 !ruby/object:Gem::Requirement
+  requirement: &70233688935720 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70348306284860
+  version_requirements: *70233688935720
+- !ruby/object:Gem::Dependency
+  name: rgeo
+  requirement: &70233688935020 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *70233688935020
+- !ruby/object:Gem::Dependency
+  name: rgeo-geojson
+  requirement: &70233688934360 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: *70233688934360
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70348306284240 !ruby/object:Gem::Requirement
+  requirement: &70233688933680 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.6.0
   type: :development
   prerelease: false
-  version_requirements: *70348306284240
+  version_requirements: *70233688933680
 - !ruby/object:Gem::Dependency
   name: mocha
-  requirement: &70348306283580 !ruby/object:Gem::Requirement
+  requirement: &70233688932800 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.0
   type: :development
   prerelease: false
-  version_requirements: *70348306283580
+  version_requirements: *70233688932800
 - !ruby/object:Gem::Dependency
   name: ruby-debug19
-  requirement: &70348306282880 !ruby/object:Gem::Requirement
+  requirement: &70233688931720 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.11.6
   type: :development
   prerelease: false
-  version_requirements: *70348306282880
+  version_requirements: *70233688931720
 description: Import CSV, SHP, and other files with data into a PostgreSQL table
 email:
 - andrew@vizzuality.com
@@ -137,7 +160,6 @@ files:
 - README.md
 - Rakefile
 - cartodb-importer.gemspec
-- clubbing.shp
 - lib/cartodb-exporter.rb
 - lib/cartodb-exporter/exporter.rb
 - lib/cartodb-exporter/version.rb