cartodb-importer 0.2.11 → 0.2.12

data/.gitignore

@@ -5,3 +5,4 @@ pkg/*
 misc/*.pyc
 bin/*
 build/*
+*.DS_Store

data/lib/cartodb-importer/importer.rb

@@ -3,7 +3,7 @@
 module CartoDB
   class Importer
     RESERVED_COLUMN_NAMES = %W{ oid tableoid xmin cmin xmax cmax ctid }
-    SUPPORTED_FORMATS = %W{ .csv .shp .ods .xls .xlsx .tif .tiff }
+    SUPPORTED_FORMATS = %W{ .csv .shp .ods .xls .xlsx .tif .tiff .kml .kmz .js .json}
     
     class << self
       attr_accessor :debug
@@ -11,7 +11,7 @@ module CartoDB
     @@debug = true
     
     attr_accessor :import_from_file,:import_from_url, :suggested_name,
-                  :ext, :db_configuration, :db_connection
+                  :ext, :db_configuration, :db_connection, :append_to_table
                   
     attr_reader :table_created, :force_name
 
@@ -22,9 +22,9 @@ module CartoDB
       if !options[:import_from_url].blank?
         #download from internet first
         potential_name = File.basename(options[:import_from_url])
-        curl_cmd = "curl -0 \"#{options[:import_from_url]}\" > /tmp/#{potential_name}"
-        #log curl_cmd
-        `#{curl_cmd}`
+        wget_cmd = "wget \"#{options[:import_from_url]}\" -O /tmp/#{potential_name}"
+        #log wget_cmd
+        `#{wget_cmd}`
         @import_from_file = "/tmp/#{potential_name}"
       else
         @import_from_file = options[:import_from_file]
@@ -36,7 +36,12 @@ module CartoDB
       @db_configuration[:port] ||= 5432
       @db_configuration[:host] ||= '127.0.0.1'
       @db_connection = Sequel.connect("postgres://#{@db_configuration[:username]}:#{@db_configuration[:password]}@#{@db_configuration[:host]}:#{@db_configuration[:port]}/#{@db_configuration[:database]}")
-
+      unless options[:append_to_table].nil?
+        @append_to_table = options[:append_to_table]
+      else
+        @append_to_table = nil
+      end
+        
       unless options[:suggested_name].nil? || options[:suggested_name].blank?
         @force_name = true
         @suggested_name = get_valid_name(options[:suggested_name])
@@ -82,7 +87,8 @@ module CartoDB
       psql_bin_path = `which psql`.strip
       
       entries = []
-      if @ext == '.zip'
+      #if @ext == '.zip'
+      if %W{ .zip .kmz }.include?(@ext)
         log "Importing zip file: #{path}"
         Zip::ZipFile.foreach(path) do |entry|
           name = entry.name.split('/').last
@@ -126,6 +132,59 @@ module CartoDB
         path = @import_from_file.path
       end
       
+      if %W{ .kmz .kml .json .js }.include?(@ext)
+        ogr2ogr_bin_path = `which ogr2ogr`.strip
+        ogr2ogr_command = %Q{#{ogr2ogr_bin_path} -f "ESRI Shapefile" #{path}.shp #{path}}
+        out = `#{ogr2ogr_command}`
+        
+        if 0 < out.strip.length
+          runlog.stdout << out
+        end
+        
+        if File.file?("#{path}.shp")
+          path = "#{path}.shp"
+          @ext = '.shp'
+        else
+          runlog.err << "failed to create shp file from kml"
+        end
+      end
+      
+      if %W{ .exxxxppp }.include?(@ext)
+        
+        ogr2ogr_bin_path = `which ogr2ogr`.strip
+        ogr2ogr_command = %Q{#{ogr2ogr_bin_path} -f "PostgreSQL" PG:"host=#{@db_configuration[:host]} port=#{@db_configuration[:port]} user=#{@db_configuration[:username]} dbname=#{@db_configuration[:database]}" #{path} -nln #{@suggested_name}}
+          
+        out = `#{ogr2ogr_command}`
+        if 0 < out.strip.length
+          runlog.stdout << out
+        end
+        
+        # Check if the file had data, if not rise an error because probably something went wrong
+        if @db_connection["SELECT * from #{@suggested_name} LIMIT 1"].first.nil?
+          runlog.err << "Empty table"
+          raise "Empty table"
+        end
+        
+        # Sanitize column names where needed
+        column_names = @db_connection.schema(@suggested_name).map{ |s| s[0].to_s }
+        need_sanitizing = column_names.each do |column_name|
+          if column_name != column_name.sanitize_column_name
+            @db_connection.run("ALTER TABLE #{@suggested_name} RENAME COLUMN \"#{column_name}\" TO #{column_name.sanitize_column_name}")
+          end
+        end
+        
+        @table_created = true
+        
+        FileUtils.rm_rf(path)
+        rows_imported = @db_connection["SELECT count(*) as count from #{@suggested_name}"].first[:count]
+        
+        return OpenStruct.new({
+          :name => @suggested_name, 
+          :rows_imported => rows_imported,
+          :import_type => import_type,
+          :log => runlog
+          })
+      end
       if @ext == '.csv'
         
         ogr2ogr_bin_path = `which ogr2ogr`.strip
@@ -163,12 +222,12 @@ module CartoDB
           })
       end
       if @ext == '.shp'
-        
+        log "processing shp"
         shp2pgsql_bin_path = `which shp2pgsql`.strip
 
         host = @db_configuration[:host] ? "-h #{@db_configuration[:host]}" : ""
         port = @db_configuration[:port] ? "-p #{@db_configuration[:port]}" : ""
-        #@suggested_name = get_valid_name(File.basename(path).tr('.','_').downcase.sanitize) unless @force_name
+        
         random_table_name = "importing_#{Time.now.to_i}_#{@suggested_name}"
         
         normalizer_command = "#{python_bin_path} -Wignore #{File.expand_path("../../../misc/shp_normalizer.py", __FILE__)} #{path} #{random_table_name}"
@@ -191,8 +250,10 @@ module CartoDB
         
         if shp_args_command[1] != '4326'
           begin  
-            @db_connection.run("SELECT UpdateGeometrySRID('#{random_table_name}', 'the_geom', 4326)")
-            @db_connection.run("UPDATE \"#{random_table_name}\" SET the_geom = ST_Transform(the_geom, 4326)")
+            @db_connection.run("ALTER TABLE #{random_table_name} RENAME COLUMN the_geom TO the_geom_orig;")
+            geom_type = @db_connection["SELECT GeometryType(the_geom_orig) as type from #{random_table_name} LIMIT 1"].first[:type]
+            @db_connection.run("SELECT AddGeometryColumn('#{random_table_name}','the_geom',4326, '#{geom_type}', 2);")
+            @db_connection.run("UPDATE \"#{random_table_name}\" SET the_geom = ST_Transform(the_geom_orig, 4326)")
             @db_connection.run("CREATE INDEX \"#{random_table_name}_the_geom_gist\" ON \"#{random_table_name}\" USING GIST (the_geom)")
           rescue Exception => msg  
             runlog.err << msg

data/lib/cartodb-importer/version.rb

@@ -1,6 +1,6 @@
 module CartoDB
   class Importer
-    VERSION = "0.2.11"
+    VERSION = "0.2.12"
   end
 end
     

data/spec/import_spec.rb

@@ -197,6 +197,46 @@ describe CartoDB::Importer do
     end
   end
   
+  describe "#KML" do
+    it "should import KML file rmnp.kml" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/rmnp.kml", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'rmnp'
+      result.rows_imported.should == 1
+      result.import_type.should == '.kml'
+    end
+    it "should import KML file rmnp.zip" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/rmnp.zip", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'rmnp'
+      result.rows_imported.should == 1
+      result.import_type.should == '.kml'
+    end
+    it "should import KMZ file rmnp.kmz" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/rmnp.kmz", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'rmnp'
+      result.rows_imported.should == 1
+      result.import_type.should == '.kml'
+    end
+  end
+  describe "#GeoJSON" do
+    it "should import GeoJSON file simple.json" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/simple.json", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'simple'
+      result.rows_imported.should == 11
+      result.import_type.should == '.json'
+    end
+  end
   describe "#SHP" do
     it "should import a SHP file in the given database in a table named like the file" do
       options = { :import_from_file => File.expand_path("../support/data/EjemploVizzuality.zip", __FILE__),
@@ -227,6 +267,7 @@ describe CartoDB::Importer do
       result.rows_imported.should == 246
       result.import_type.should == '.shp'
     end
+  
 
     it "should import SHP file TM_WORLD_BORDERS_SIMPL-0.3.zip but set the given name" do
       importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/TM_WORLD_BORDERS_SIMPL-0.3.zip", __FILE__),
@@ -282,4 +323,14 @@ describe CartoDB::Importer do
     end
   end  
   
+  describe "Import from Simon file" do
+    it "should import a shapefile from Simon" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/simon-search-spain-1297870422647.zip", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      result = importer.import!
+      #result.rows_imported.should == 312
+      result.import_type.should == '.shp'
+    end
+  end  
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cartodb-importer
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.2.12
   prerelease: 
 platform: ruby
 authors:
@@ -11,11 +11,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-20 00:00:00.000000000Z
+date: 2011-09-26 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
-  requirement: &70275501630760 !ruby/object:Gem::Requirement
+  requirement: &70120001441940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -23,10 +23,10 @@ dependencies:
         version: '0.11'
   type: :runtime
   prerelease: false
-  version_requirements: *70275501630760
+  version_requirements: *70120001441940
 - !ruby/object:Gem::Dependency
   name: sequel
-  requirement: &70275501630140 !ruby/object:Gem::Requirement
+  requirement: &70120001425220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -34,10 +34,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275501630140
+  version_requirements: *70120001425220
 - !ruby/object:Gem::Dependency
   name: roo
-  requirement: &70275501629380 !ruby/object:Gem::Requirement
+  requirement: &70120001424340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -45,10 +45,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275501629380
+  version_requirements: *70120001424340
 - !ruby/object:Gem::Dependency
   name: spreadsheet
-  requirement: &70275501616740 !ruby/object:Gem::Requirement
+  requirement: &70120001423720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -56,10 +56,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275501616740
+  version_requirements: *70120001423720
 - !ruby/object:Gem::Dependency
   name: google-spreadsheet-ruby
-  requirement: &70275501616240 !ruby/object:Gem::Requirement
+  requirement: &70120001422940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -67,10 +67,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275501616240
+  version_requirements: *70120001422940
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &70275501615640 !ruby/object:Gem::Requirement
+  requirement: &70120001421960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -78,10 +78,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275501615640
+  version_requirements: *70120001421960
 - !ruby/object:Gem::Dependency
   name: builder
-  requirement: &70275501614920 !ruby/object:Gem::Requirement
+  requirement: &70120001421380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -89,10 +89,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275501614920
+  version_requirements: *70120001421380
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70275501614360 !ruby/object:Gem::Requirement
+  requirement: &70120001420760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -100,10 +100,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70275501614360
+  version_requirements: *70120001420760
 - !ruby/object:Gem::Dependency
   name: mocha
-  requirement: &70275501613760 !ruby/object:Gem::Requirement
+  requirement: &70120001420080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -111,10 +111,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70275501613760
+  version_requirements: *70120001420080
 - !ruby/object:Gem::Dependency
   name: ruby-debug19
-  requirement: &70275501613100 !ruby/object:Gem::Requirement
+  requirement: &70120001419420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -122,7 +122,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70275501613100
+  version_requirements: *70120001419420
 description: Import CSV, SHP, and other files with data into a PostgreSQL table
 email:
 - andrew@vizzuality.com