cartodb-importer 0.2.6 → 0.2.7

data/lib/cartodb-importer/importer.rb

@@ -89,8 +89,9 @@ module CartoDB
           entry.extract("/tmp/#{name}")
         end
       end
-      
+        
       import_type = @ext
+      
       # These types of files are converted to CSV
       if %W{ .xls .xlsx .ods }.include?(@ext)
         new_path = "/tmp/#{@suggested_name}.csv"
@@ -146,7 +147,7 @@ module CartoDB
 
         host = @db_configuration[:host] ? "-h #{@db_configuration[:host]}" : ""
         port = @db_configuration[:port] ? "-p #{@db_configuration[:port]}" : ""
-        @suggested_name = get_valid_name(File.basename(path).tr('.','_').downcase.sanitize) unless @force_name
+        #@suggested_name = get_valid_name(File.basename(path).tr('.','_').downcase.sanitize) unless @force_name
         random_table_name = "importing_#{Time.now.to_i}_#{@suggested_name}"
         
         normalizer_command = "#{python_bin_path} -Wignore #{File.expand_path("../../../misc/shp_normalizer.py", __FILE__)} #{path} #{random_table_name}"
@@ -185,7 +186,7 @@ module CartoDB
         
         host = @db_configuration[:host] ? "-h #{@db_configuration[:host]}" : ""
         port = @db_configuration[:port] ? "-p #{@db_configuration[:port]}" : ""
-        @suggested_name = get_valid_name(File.basename(path).tr('.','_').downcase.sanitize) unless @force_name
+        #@suggested_name = get_valid_name(File.basename(path).tr('.','_').downcase.sanitize) unless @force_name
         random_table_name = "importing_#{Time.now.to_i}_#{@suggested_name}"
         
         gdal_command = "#{python_bin_path} -Wignore #{File.expand_path("../../../misc/srid_from_gdal.py", __FILE__)} #{path}"

data/lib/cartodb-importer/version.rb

@@ -1,6 +1,6 @@
 module CartoDB
   class Importer
-    VERSION = "0.2.6"
+    VERSION = "0.2.7"
   end
 end
     

data/misc/shp_normalizer.py

@@ -39,19 +39,21 @@ if os.path.isfile(prj_file):
     if jres['codes']:
       srid = int(jres['codes'][0]['code'])
     
-#Try to detect the encoding
-dbf = open(dbf_file, 'rb')
-db = dbfUtils.dbfreader(dbf)
-
-detector = UniversalDetector()
-for row in db:
-  detector.feed(str(row))
-  if detector.done: break
-detector.close()
-dbf.close()
-
-encoding = detector.result["encoding"]
-if encoding=="ascii":
-    encoding="LATIN1"
+try:
+    #Try to detect the encoding
+    dbf = open(dbf_file, 'rb')
+    db = dbfUtils.dbfreader(dbf)
+    detector = UniversalDetector()
+    for row in db:
+      detector.feed(str(row))
+      if detector.done: break
+    detector.close()
+    dbf.close()
+    encoding = detector.result["encoding"]
+    if encoding=="ascii":
+        encoding="LATIN1"
+except:
+    #if encoding detection fails, attempt default UTF8
+    encoding = "UTF8"
 
 print "%s,%s,%s,%s" % (srid,encoding,shp_file,name)

data/spec/import_spec.rb

@@ -205,13 +205,13 @@ describe CartoDB::Importer do
                                        
       importer = CartoDB::Importer.new(options)
       result = importer.import!
-      result.name.should == 'vizzuality_shp'
+      result.name.should == 'vizzuality'
       result.rows_imported.should == 11
       result.import_type.should == '.shp'
       
       db_connection = Sequel.connect("postgres://#{options[:username]}:#{options[:password]}@#{options[:host]}:#{options[:port]}/#{options[:database]}")
-      db_connection.tables.should include(:vizzuality_shp)
-      columns = db_connection.schema(:vizzuality_shp).map{|s| s[0].to_s}
+      db_connection.tables.should include(:vizzuality)
+      columns = db_connection.schema(:vizzuality).map{|s| s[0].to_s}
       
       expected_columns = ["gid", "subclass", "x", "y", "length", "area", "angle", "name", 
         "pid", "lot_navteq", "version_na", "vitesse_sp", "id", "nombrerest", "tipocomida", "the_geom"]
@@ -223,7 +223,7 @@ describe CartoDB::Importer do
                                        :database => "cartodb_importer_test", :username => 'postgres', :password => '',
                                        :host => 'localhost', :port => 5432
       result = importer.import!
-      result.name.should == 'tm_world_borders_simpl_0_3_shp'
+      result.name.should == 'tm_world_borders_simpl_0_3'
       result.rows_imported.should == 246
       result.import_type.should == '.shp'
     end
@@ -244,29 +244,28 @@ describe CartoDB::Importer do
                                        :database => "cartodb_importer_test", :username => 'postgres', :password => '',
                                        :host => 'localhost', :port => 5432
       result = importer.import!
-      result.name.should == 'global_elevation_simple_tif'
+      result.name.should == 'global_elevation_simple'
       result.rows_imported.should == 1500
       result.import_type.should == '.tif'
     end
   end  
   describe "Extended" do
-    it "should import 1 SHP file" do
-      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../../../vm-cartodb/extended_tests/cua2005index_sp.zip", __FILE__),
+    it "should import 2 SHP files incrementing the name of the second" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/TM_WORLD_BORDERS_SIMPL-0.3.zip", __FILE__),
                                        :database => "cartodb_importer_test", :username => 'postgres', :password => '',
                                        :host => 'localhost', :port => 5432
       result = importer.import!
-      result.name.should == 'cua2005index_sp_shp'
-      result.rows_imported.should == 4365
+      result.name.should == 'tm_world_borders_simpl_0_3'
+      #result.rows_imported.should == 4365
       result.import_type.should == '.shp'
-    end
-    it "should import 2 SHP file" do
-      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../../../vm-cartodb/extended_tests/cua2005index_utm.zip", __FILE__),
+      
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/TM_WORLD_BORDERS_SIMPL-0.3.zip", __FILE__),
                                        :database => "cartodb_importer_test", :username => 'postgres', :password => '',
                                        :host => 'localhost', :port => 5432
-      result = importer.import!
-      result.name.should == 'cua2005index_utm_shp'
-      result.rows_imported.should == 4528
-      result.import_type.should == '.shp'
+      result2 = importer.import!
+      result2.name.should == 'tm_world_borders_simpl_0_4'
+      #result.rows_imported.should == 4365
+      result2.import_type.should == '.shp'
     end
   end  
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cartodb-importer
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
   prerelease: 
 platform: ruby
 authors:
@@ -15,7 +15,7 @@ date: 2011-09-13 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
-  requirement: &70183805156060 !ruby/object:Gem::Requirement
+  requirement: &70204688871100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -23,10 +23,10 @@ dependencies:
         version: '0.11'
   type: :runtime
   prerelease: false
-  version_requirements: *70183805156060
+  version_requirements: *70204688871100
 - !ruby/object:Gem::Dependency
   name: sequel
-  requirement: &70183805155580 !ruby/object:Gem::Requirement
+  requirement: &70204688870460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -34,10 +34,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70183805155580
+  version_requirements: *70204688870460
 - !ruby/object:Gem::Dependency
   name: roo
-  requirement: &70183805155120 !ruby/object:Gem::Requirement
+  requirement: &70204688869740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -45,10 +45,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70183805155120
+  version_requirements: *70204688869740
 - !ruby/object:Gem::Dependency
   name: spreadsheet
-  requirement: &70183805154700 !ruby/object:Gem::Requirement
+  requirement: &70204688869160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -56,10 +56,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70183805154700
+  version_requirements: *70204688869160
 - !ruby/object:Gem::Dependency
   name: google-spreadsheet-ruby
-  requirement: &70183805154080 !ruby/object:Gem::Requirement
+  requirement: &70204688868520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -67,10 +67,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70183805154080
+  version_requirements: *70204688868520
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &70183805153440 !ruby/object:Gem::Requirement
+  requirement: &70204688867880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -78,10 +78,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70183805153440
+  version_requirements: *70204688867880
 - !ruby/object:Gem::Dependency
   name: builder
-  requirement: &70183805152820 !ruby/object:Gem::Requirement
+  requirement: &70204688867260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -89,10 +89,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70183805152820
+  version_requirements: *70204688867260
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70183805152180 !ruby/object:Gem::Requirement
+  requirement: &70204688866620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -100,10 +100,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70183805152180
+  version_requirements: *70204688866620
 - !ruby/object:Gem::Dependency
   name: mocha
-  requirement: &70183805151580 !ruby/object:Gem::Requirement
+  requirement: &70204688865980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -111,10 +111,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70183805151580
+  version_requirements: *70204688865980
 - !ruby/object:Gem::Dependency
   name: ruby-debug19
-  requirement: &70183805150960 !ruby/object:Gem::Requirement
+  requirement: &70204688865360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -122,7 +122,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70183805150960
+  version_requirements: *70204688865360
 description: Import CSV, SHP, and other files with data into a PostgreSQL table
 email:
 - andrew@vizzuality.com