cartodb-importer 0.2.8 → 0.2.9

data/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2011, Vizzuality 
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. All advertising materials mentioning features or use of this software
+   must display the following acknowledgement:
+   This product includes software developed by Vizzuality.
+4. Neither the name of Vizzuality nor the
+   names of its contributors may be used to endorse or promote products
+   derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -9,9 +9,9 @@ CartoDB importer is a Ruby gem that makes your life easier when importing data f
   
 ## Installation and dependencies ##
 
-To install Ruby dependencies just install `bundler` gem and run the command `bundle install` in your shell.
+To install Ruby dependencies just install `bundler` the gem and run the command `bundle install` in your shell.
 
-There are also some dependencies of external Python libraries (WTF!). You should install `pip` before:
+There are also some dependencies of external Python libraries. You should install `pip` before:
 
   - In Debian / Ubuntu: `apt-get install python-pip`
   

data/lib/cartodb-importer/importer.rb

@@ -10,7 +10,7 @@ module CartoDB
     end
     @@debug = true
     
-    attr_accessor :import_from_file, :suggested_name,
+    attr_accessor :import_from_file,:import_from_url, :suggested_name,
                   :ext, :db_configuration, :db_connection
                   
     attr_reader :table_created, :force_name
@@ -18,7 +18,18 @@ module CartoDB
     def initialize(options = {})
       @@debug = options[:debug] if options[:debug]
       @table_created = nil
-      @import_from_file = options[:import_from_file]
+      
+      if !options[:import_from_url].blank?
+        #download from internet first
+        potential_name = File.basename(options[:import_from_url])
+        curl_cmd = "curl -0 \"#{options[:import_from_url]}\" > /tmp/#{potential_name}"
+        #log curl_cmd
+        `#{curl_cmd}`
+        @import_from_file = "/tmp/#{potential_name}"
+      else
+        @import_from_file = options[:import_from_file]
+      end
+      
       raise "import_from_file value can't be nil" if @import_from_file.nil?
 
       @db_configuration = options.slice(:database, :username, :password, :host, :port)
@@ -91,6 +102,10 @@ module CartoDB
       end
         
       import_type = @ext
+      runlog = OpenStruct.new
+      runlog.log = Array.new
+      runlog.stdout = Array.new
+      runlog.err = Array.new
       
       # These types of files are converted to CSV
       if %W{ .xls .xlsx .ods }.include?(@ext)
@@ -103,6 +118,7 @@ module CartoDB
           when '.ods'
             Openoffice.new(path)
           else
+            runlog.log << "Don't know how to open file #{new_path}"
             raise ArgumentError, "Don't know how to open file #{new_path}"
         end.to_csv(new_path)
         @import_from_file = File.open(new_path,'r')
@@ -114,11 +130,15 @@ module CartoDB
         
         ogr2ogr_bin_path = `which ogr2ogr`.strip
         ogr2ogr_command = %Q{#{ogr2ogr_bin_path} -f "PostgreSQL" PG:"host=#{@db_configuration[:host]} port=#{@db_configuration[:port]} user=#{@db_configuration[:username]} dbname=#{@db_configuration[:database]}" #{path} -nln #{@suggested_name}}
-        
-        output = `#{ogr2ogr_command} &> /dev/null`
+          
+        out = `#{ogr2ogr_command}`
+        if 0 < out.strip.length
+          runlog.stdout << out
+        end
         
         # Check if the file had data, if not rise an error because probably something went wrong
         if @db_connection["SELECT * from #{@suggested_name} LIMIT 1"].first.nil?
+          runlog.err << "Empty table"
           raise "Empty table"
         end
         
@@ -134,13 +154,13 @@ module CartoDB
         
         FileUtils.rm_rf(path)
         rows_imported = @db_connection["SELECT count(*) as count from #{@suggested_name}"].first[:count]
-
+        
         return OpenStruct.new({
           :name => @suggested_name, 
           :rows_imported => rows_imported,
-          :import_type => import_type
+          :import_type => import_type,
+          :log => runlog
           })
-        
       end
       if @ext == '.shp'
         
@@ -152,71 +172,101 @@ module CartoDB
         random_table_name = "importing_#{Time.now.to_i}_#{@suggested_name}"
         
         normalizer_command = "#{python_bin_path} -Wignore #{File.expand_path("../../../misc/shp_normalizer.py", __FILE__)} #{path} #{random_table_name}"
-        shp_args_command = `#{normalizer_command}`.split( /, */, 4 )
-
-        #print "-e -i -I -g the_geom -W %s %s %s %s" %(srid,encoding,shp_file,name,flag)
-
+        out = `#{normalizer_command}`
+        shp_args_command = out.split( /, */, 4 )
+        
         if shp_args_command.length != 4
-        raise "Error running python shp_normalizer script: #{normalizer_command}"
+          runlog.log << "Error running python shp_normalizer script: #{normalizer_command}"
+          runlog.stdout << out
+          raise "Error running python shp_normalizer script: #{normalizer_command}"
         end
-        full_shp_command = "#{shp2pgsql_bin_path} -s #{shp_args_command[0]} -e -i -I -g the_geom -W #{shp_args_command[1]} #{shp_args_command[2]} #{shp_args_command[3].strip} | #{psql_bin_path} #{host} #{port} -U #{@db_configuration[:username]} -w -d #{@db_configuration[:database]}"
+        
+        full_shp_command = "#{shp2pgsql_bin_path} -s #{shp_args_command[0]} -e -i -g the_geom -W #{shp_args_command[1]} #{shp_args_command[2]} #{shp_args_command[3].strip} | #{psql_bin_path} #{host} #{port} -U #{@db_configuration[:username]} -w -d #{@db_configuration[:database]}"
         log "Running shp2pgsql: #{full_shp_command}"
-        %x[#{full_shp_command}]
+        
+        out = `#{full_shp_command}`
+        if 0 < out.strip.length
+          runlog.stdout << out
+        end
+        
         if shp_args_command[1] != '4326'
-          
-          @db_connection.run("SELECT UpdateGeometrySRID('#{random_table_name}', 'the_geom', 4326)")
-          @db_connection.run("UPDATE #{random_table_name} SET the_geom = ST_Transform(the_geom, 4326)")
+          begin  
+            @db_connection.run("SELECT UpdateGeometrySRID('#{random_table_name}', 'the_geom', 4326)")
+            @db_connection.run("UPDATE \"#{random_table_name}\" SET the_geom = ST_Transform(the_geom, 4326)")
+            @db_connection.run("CREATE INDEX \"#{random_table_name}_the_geom_gist\" ON \"#{random_table_name}\" USING GIST (the_geom)")
+          rescue Exception => msg  
+            runlog.err << msg
+          end  
         end
-        @db_connection.run("CREATE TABLE #{@suggested_name} AS SELECT * FROM #{random_table_name}")
-        @db_connection.run("DROP TABLE #{random_table_name}")
-        @table_created = true
         
+        begin
+          @db_connection.run("ALTER TABLE \"#{random_table_name}\" RENAME TO \"#{@suggested_name}\"")
+          @table_created = true
+        rescue Exception => msg  
+          runlog.err << msg
+        end  
         entries.each{ |e| FileUtils.rm_rf(e) } if entries.any?
-        rows_imported = @db_connection["SELECT count(*) as count from #{@suggested_name}"].first[:count]
+        rows_imported = @db_connection["SELECT count(*) as count from \"#{@suggested_name}\""].first[:count]
         @import_from_file.unlink
 
         return OpenStruct.new({
-        :name => @suggested_name, 
-        :rows_imported => rows_imported,
-        :import_type => import_type
+          :name => @suggested_name, 
+          :rows_imported => rows_imported,
+          :import_type => import_type,
+          :log => runlog
         })
       end
       if %W{ .tif .tiff }.include?(@ext)  
         log "Importing raster file: #{path}"
+        
         raster2pgsql_bin_path = `which raster2pgsql.py`.strip
         
         host = @db_configuration[:host] ? "-h #{@db_configuration[:host]}" : ""
         port = @db_configuration[:port] ? "-p #{@db_configuration[:port]}" : ""
-        #@suggested_name = get_valid_name(File.basename(path).tr('.','_').downcase.sanitize) unless @force_name
+        
         random_table_name = "importing_#{Time.now.to_i}_#{@suggested_name}"
         
         gdal_command = "#{python_bin_path} -Wignore #{File.expand_path("../../../misc/srid_from_gdal.py", __FILE__)} #{path}"
         rast_srid_command = `#{gdal_command}`.strip
         
+        if 0 < rast_srid_command.strip.length
+          runlog.stdout << rast_srid_command
+        end
+        
+        
         log "SRID : #{rast_srid_command}"
         
         blocksize = "180x180"
         full_rast_command = "#{raster2pgsql_bin_path} -I -s #{rast_srid_command.strip} -k #{blocksize} -t  #{random_table_name} -r #{path} | #{psql_bin_path} #{host} #{port} -U #{@db_configuration[:username]} -w -d #{@db_configuration[:database]}"
         log "Running raster2pgsql: #{raster2pgsql_bin_path}  #{full_rast_command}"
-        %x[#{full_rast_command}]
+        out = `#{full_rast_command}`
+        if 0 < out.strip.length
+          runlog.stdout << out
+        end
         
-        @db_connection.run("CREATE TABLE #{@suggested_name} AS SELECT * FROM #{random_table_name}")
-        @db_connection.run("DROP TABLE #{random_table_name}")
+        begin
+          @db_connection.run("CREATE TABLE \"#{@suggested_name}\" AS SELECT * FROM \"#{random_table_name}\"")
+          @db_connection.run("DROP TABLE \"#{random_table_name}\"")
+          @table_created = true
+        rescue Exception => msg  
+          runlog.err << msg
+        end  
         
         entries.each{ |e| FileUtils.rm_rf(e) } if entries.any?
-        rows_imported = @db_connection["SELECT count(*) as count from #{@suggested_name}"].first[:count]
+        rows_imported = @db_connection["SELECT count(*) as count from \"#{@suggested_name}\""].first[:count]
         @import_from_file.unlink
         
         @table_created = true
         
         entries.each{ |e| FileUtils.rm_rf(e) } if entries.any?
-        rows_imported = @db_connection["SELECT count(*) as count from #{@suggested_name}"].first[:count]
+        rows_imported = @db_connection["SELECT count(*) as count from \"#{@suggested_name}\""].first[:count]
         @import_from_file.unlink
         
         return OpenStruct.new({
           :name => @suggested_name, 
           :rows_imported => rows_imported,
-          :import_type => import_type
+          :import_type => import_type,
+          :log => runlog
         })
         
       end

data/lib/cartodb-importer/version.rb

@@ -1,6 +1,6 @@
 module CartoDB
   class Importer
-    VERSION = "0.2.8"
+    VERSION = "0.2.9"
   end
 end
     

data/spec/import_spec.rb

@@ -64,7 +64,7 @@ describe CartoDB::Importer do
                                      :database => "cartodb_importer_test", :username => 'postgres', :password => '',
                                      :host => 'localhost', :port => 5432, :suggested_name => 'prefered_name'
     result = importer.import!
-    result.name.should == 'prefered_name_2'
+    result.name.should == 'prefered_name_1'
     result.rows_imported.should == 1998
     result.import_type.should == '.csv'
   end
@@ -182,14 +182,6 @@ describe CartoDB::Importer do
       result.name.should == 'estaciones2'
       result.rows_imported.should == 30
       result.import_type.should == '.csv'
-      
-      importer2 = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/estaciones2.csv", __FILE__),
-                                     :database => "cartodb_importer_test", :username => 'postgres', :password => '',
-                                     :host => 'localhost', :port => 5432, :suggested_name => 'estaciones2'
-      result2 = importer2.import!
-      result2.name.should == 'estaciones2_0'
-      result2.rows_imported.should == 30
-      result2.import_type.should == '.csv'
     end
   end
   
@@ -266,14 +258,28 @@ describe CartoDB::Importer do
       result.name.should == 'tm_world_borders_simpl_0_3'
       #result.rows_imported.should == 4365
       result.import_type.should == '.shp'
-      
-      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/TM_WORLD_BORDERS_SIMPL-0.3.zip", __FILE__),
+    end
+  end  
+  describe "Natural Earth Polygons" do
+    it "should import Natural Earth Polygons" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/110m-glaciated-areas.zip", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      result = importer.import!
+      #result.rows_imported.should == 4365
+      result.import_type.should == '.shp'
+    end
+  end  
+  
+  describe "Import from URL" do
+    it "should import a shapefile from NaturalEarthData.com" do
+      importer = CartoDB::Importer.new :import_from_url => "http://www.nacis.org/naturalearth/10m/cultural/10m_parks_and_protected_areas.zip",
                                        :database => "cartodb_importer_test", :username => 'postgres', :password => '',
                                        :host => 'localhost', :port => 5432
-      result2 = importer.import!
-      result2.name.should == 'tm_world_borders_simpl_0_3_0'
+      result = importer.import!
       #result.rows_imported.should == 4365
-      result2.import_type.should == '.shp'
+      result.import_type.should == '.shp'
     end
   end  
+  
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cartodb-importer
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.9
   prerelease: 
 platform: ruby
 authors:
@@ -11,11 +11,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-14 00:00:00.000000000Z
+date: 2011-09-19 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
-  requirement: &70263347758260 !ruby/object:Gem::Requirement
+  requirement: &70265809904660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -23,10 +23,10 @@ dependencies:
         version: '0.11'
   type: :runtime
   prerelease: false
-  version_requirements: *70263347758260
+  version_requirements: *70265809904660
 - !ruby/object:Gem::Dependency
   name: sequel
-  requirement: &70263347757840 !ruby/object:Gem::Requirement
+  requirement: &70265809904080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -34,10 +34,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70263347757840
+  version_requirements: *70265809904080
 - !ruby/object:Gem::Dependency
   name: roo
-  requirement: &70263347757380 !ruby/object:Gem::Requirement
+  requirement: &70265809903620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -45,10 +45,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70263347757380
+  version_requirements: *70265809903620
 - !ruby/object:Gem::Dependency
   name: spreadsheet
-  requirement: &70263347756940 !ruby/object:Gem::Requirement
+  requirement: &70265809903200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -56,10 +56,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70263347756940
+  version_requirements: *70265809903200
 - !ruby/object:Gem::Dependency
   name: google-spreadsheet-ruby
-  requirement: &70263347756360 !ruby/object:Gem::Requirement
+  requirement: &70265809902760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -67,10 +67,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70263347756360
+  version_requirements: *70265809902760
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &70263347755840 !ruby/object:Gem::Requirement
+  requirement: &70265809902340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -78,10 +78,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70263347755840
+  version_requirements: *70265809902340
 - !ruby/object:Gem::Dependency
   name: builder
-  requirement: &70263347755180 !ruby/object:Gem::Requirement
+  requirement: &70265809901920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -89,10 +89,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70263347755180
+  version_requirements: *70265809901920
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70263347754620 !ruby/object:Gem::Requirement
+  requirement: &70265809901500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -100,10 +100,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70263347754620
+  version_requirements: *70265809901500
 - !ruby/object:Gem::Dependency
   name: mocha
-  requirement: &70263347754140 !ruby/object:Gem::Requirement
+  requirement: &70265809901080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -111,10 +111,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70263347754140
+  version_requirements: *70265809901080
 - !ruby/object:Gem::Dependency
   name: ruby-debug19
-  requirement: &70263347753660 !ruby/object:Gem::Requirement
+  requirement: &70265809900620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -122,7 +122,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70263347753660
+  version_requirements: *70265809900620
 description: Import CSV, SHP, and other files with data into a PostgreSQL table
 email:
 - andrew@vizzuality.com
@@ -133,6 +133,7 @@ files:
 - .gitignore
 - .rvmrc
 - Gemfile
+- LICENSE
 - README.md
 - Rakefile
 - cartodb-importer.gemspec