cartodb-importer 0.2.2

data/lib/cartodb-importer.rb

@@ -0,0 +1,14 @@
+# coding: UTF-8
+
+require 'rubygems'
+require 'bundler'
+Bundler.setup
+
+require 'roo'
+require 'csv'
+require 'tempfile'
+require 'ostruct'
+require 'cartodb-importer/importer'
+require 'core_ext/string'
+require 'core_ext/hash'
+require 'core_ext/blank'

data/lib/core_ext/blank.rb

@@ -0,0 +1,3 @@
+class NilClass
+  def blank?; true end
+end

data/lib/core_ext/hash.rb

@@ -0,0 +1,10 @@
+# coding: UTF-8
+
+class Hash
+  def slice(*keys)
+    keys = keys.map! { |key| convert_key(key) } if respond_to?(:convert_key)
+    hash = self.class.new
+    keys.each { |k| hash[k] = self[k] if has_key?(k) }
+    hash
+  end
+end

data/lib/core_ext/string.rb

@@ -0,0 +1,91 @@
+# coding: UTF-8
+
+class String
+  
+  POSTGRESQL_RESERVED_WORDS = %W{ ALL ANALYSE ANALYZE AND ANY ARRAY AS ASC ASYMMETRIC AUTHORIZATION BETWEEN BINARY BOTH CASE CAST 
+                                  CHECK COLLATE COLUMN CONSTRAINT CREATE CROSS CURRENT_DATE CURRENT_ROLE CURRENT_TIME CURRENT_TIMESTAMP 
+                                  CURRENT_USER DEFAULT DEFERRABLE DESC DISTINCT DO ELSE END EXCEPT FALSE FOR FOREIGN FREEZE FROM FULL
+                                  GRANT GROUP HAVING ILIKE IN INITIALLY INNER INTERSECT INTO IS ISNULL JOIN LEADING LEFT LIKE LIMIT LOCALTIME 
+                                  LOCALTIMESTAMP NATURAL NEW NOT NOTNULL NULL OFF OFFSET OLD ON ONLY OR ORDER OUTER OVERLAPS PLACING PRIMARY 
+                                  REFERENCES RIGHT SELECT SESSION_USER SIMILAR SOME SYMMETRIC TABLE THEN TO TRAILING TRUE UNION UNIQUE USER 
+                                  USING VERBOSE WHEN WHERE }
+  
+  
+  def blank?
+    self !~ /\S/
+  end
+  
+  def self.random(length=10)
+  ('a'..'z').sort_by {rand}[0,length].join
+  end
+
+  def normalize
+    str = self.downcase
+    return '' if str.blank?
+    n = str.force_encoding("UTF-8")
+    n.gsub!(/[àáâãäåāă]/,   'a')
+    n.gsub!(/æ/,            'ae')
+    n.gsub!(/[ďđ]/,          'd')
+    n.gsub!(/[çćčĉċ]/,       'c')
+    n.gsub!(/[èéêëēęěĕė]/,   'e')
+    n.gsub!(/ƒ/,             'f')
+    n.gsub!(/[ĝğġģ]/,        'g')
+    n.gsub!(/[ĥħ]/,          'h')
+    n.gsub!(/[ììíîïīĩĭ]/,    'i')
+    n.gsub!(/[įıijĵ]/,        'j')
+    n.gsub!(/[ķĸ]/,          'k')
+    n.gsub!(/[łľĺļŀ]/,       'l')
+    n.gsub!(/[ñńňņʼnŋ]/,      'n')
+    n.gsub!(/[òóôõöøōőŏŏ]/,  'o')
+    n.gsub!(/œ/,            'oe')
+    n.gsub!(/ą/,             'q')
+    n.gsub!(/[ŕřŗ]/,         'r')
+    n.gsub!(/[śšşŝș]/,       's')
+    n.gsub!(/[ťţŧț]/,        't')
+    n.gsub!(/[ùúûüūůűŭũų]/,  'u')
+    n.gsub!(/ŵ/,             'w')
+    n.gsub!(/[ýÿŷ]/,         'y')
+    n.gsub!(/[žżź]/,         'z')
+    n.gsub!(/[ÀÁÂÃÄÅĀĂ]/i,    'A')
+    n.gsub!(/Æ/i,            'AE')
+    n.gsub!(/[ĎĐ]/i,          'D')
+    n.gsub!(/[ÇĆČĈĊ]/i,       'C')
+    n.gsub!(/[ÈÉÊËĒĘĚĔĖ]/i,   'E')
+    n.gsub!(/Ƒ/i,             'F')
+    n.gsub!(/[ĜĞĠĢ]/i,        'G')
+    n.gsub!(/[ĤĦ]/i,          'H')
+    n.gsub!(/[ÌÌÍÎÏĪĨĬ]/i,    'I')
+    n.gsub!(/[IJĴ]/i,          'J')
+    n.gsub!(/[Ķĸ]/i,          'J')
+    n.gsub!(/[ŁĽĹĻĿ]/i,       'L')
+    n.gsub!(/[ÑŃŇŅʼnŊ]/i,      'M')
+    n.gsub!(/[ÒÓÔÕÖØŌŐŎŎ]/i,  'N')
+    n.gsub!(/Œ/i,            'OE')
+    n.gsub!(/Ą/i,             'Q')
+    n.gsub!(/[ŔŘŖ]/i,         'R')
+    n.gsub!(/[ŚŠŞŜȘ]/i,       'S')
+    n.gsub!(/[ŤŢŦȚ]/i,        'T')
+    n.gsub!(/[ÙÚÛÜŪŮŰŬŨŲ]/i,  'U')
+    n.gsub!(/Ŵ/i,             'W')
+    n.gsub!(/[ÝŸŶ]/i,         'Y')
+    n.gsub!(/[ŽŻŹ]/i,         'Z')
+    n
+  end
+
+  def sanitize
+   return if self.blank?
+   self.gsub(/<[^>]+>/m,'').normalize.downcase.gsub(/&.+?;/,'-').
+        gsub(/[^a-z0-9 _-]/,'-').strip.gsub(/\s+/,'-').gsub(/-+/,'-').
+        gsub(/-/,' ').strip.gsub(/ /,'-').gsub(/-/,'_')
+  end
+
+  def sanitize_column_name
+    temporal_name = self.sanitize
+    if temporal_name !~ /^[a-zA-Z_]/ || POSTGRESQL_RESERVED_WORDS.include?(self.upcase)
+      return '_' + temporal_name
+    else
+      temporal_name
+    end
+  end
+
+end

data/misc/csv_normalizer.py

@@ -0,0 +1,27 @@
+import sys
+import brewery.ds as ds
+import brewery.dq as dq
+from chardet.universaldetector import UniversalDetector
+
+filename = sys.argv[1]
+
+detector = UniversalDetector()
+for line in file(filename, 'rb'):
+    detector.feed(line)
+    if detector.done: break
+detector.close()
+
+src = ds.CSVDataSource(filename, read_header = True, encoding=detector.result["encoding"], delimiter=',' )
+src.initialize()
+if len(src.field_names) == 1:
+  src.finalize()
+  src = ds.CSVDataSource(filename, read_header = True, encoding=detector.result["encoding"], delimiter=';' )
+  src.initialize()
+  
+out = ds.CSVDataTarget(sys.stdout, encoding='utf-8')
+out.fields = ds.fieldlist(src.field_names)
+out.initialize()
+for record in src.records():
+  out.append(record)
+src.finalize()
+out.finalize()

data/misc/dbfUtils.py

@@ -0,0 +1,113 @@
+import struct, datetime, decimal, itertools
+
+def dbfreader(f):
+    """Returns an iterator over records in a Xbase DBF file.
+
+    The first row returned contains the field names.
+    The second row contains field specs: (type, size, decimal places).
+    Subsequent rows contain the data records.
+    If a record is marked as deleted, it is skipped.
+
+    File should be opened for binary reads.
+
+    """
+    # See DBF format spec at:
+    #     http://www.pgts.com.au/download/public/xbase.htm#DBF_STRUCT
+
+    numrec, lenheader = struct.unpack('<xxxxLH22x', f.read(32))    
+    numfields = (lenheader - 33) // 32
+
+    fields = []
+    for fieldno in xrange(numfields):
+        name, typ, size, deci = struct.unpack('<11sc4xBB14x', f.read(32))
+        name = name.replace('\0', '')       # eliminate NULs from string   
+        fields.append((name, typ, size, deci))
+    yield [field[0] for field in fields]
+    yield [tuple(field[1:]) for field in fields]
+
+    terminator = f.read(1)
+    assert terminator == '\r'
+
+    fields.insert(0, ('DeletionFlag', 'C', 1, 0))
+    fmt = ''.join(['%ds' % fieldinfo[2] for fieldinfo in fields])
+    fmtsiz = struct.calcsize(fmt)
+    for i in xrange(numrec):
+        record = struct.unpack(fmt, f.read(fmtsiz))
+        if record[0] != ' ':
+            continue                        # deleted record
+        result = []
+        for (name, typ, size, deci), value in itertools.izip(fields, record):
+            if name == 'DeletionFlag':
+                continue
+            if typ == "N":
+                value = value.replace('\0', '').lstrip()
+                if value == '':
+                    value = 0
+                elif deci:
+                    value = decimal.Decimal(value)
+                else:
+                    value = value
+            elif typ == 'D':
+                y, m, d = int(value[:4]), int(value[4:6]), int(value[6:8])
+                value = datetime.date(y, m, d)
+            elif typ == 'L':
+                value = (value in 'YyTt' and 'T') or (value in 'NnFf' and 'F') or '?'
+            result.append(value)
+        yield result
+
+
+def dbfwriter(f, fieldnames, fieldspecs, records):
+    """ Return a string suitable for writing directly to a binary dbf file.
+
+    File f should be open for writing in a binary mode.
+
+    Fieldnames should be no longer than ten characters and not include \x00.
+    Fieldspecs are in the form (type, size, deci) where
+        type is one of:
+            C for ascii character data
+            M for ascii character memo data (real memo fields not supported)
+            D for datetime objects
+            N for ints or decimal objects
+            L for logical values 'T', 'F', or '?'
+        size is the field width
+        deci is the number of decimal places in the provided decimal object
+    Records can be an iterable over the records (sequences of field values).
+    
+    """
+    # header info
+    ver = 3
+    now = datetime.datetime.now()
+    yr, mon, day = now.year-1900, now.month, now.day
+    numrec = len(records)
+    numfields = len(fieldspecs)
+    lenheader = numfields * 32 + 33
+    lenrecord = sum(field[1] for field in fieldspecs) + 1
+    hdr = struct.pack('<BBBBLHH20x', ver, yr, mon, day, numrec, lenheader, lenrecord)
+    f.write(hdr)
+                      
+    # field specs
+    for name, (typ, size, deci) in itertools.izip(fieldnames, fieldspecs):
+        name = name.ljust(11, '\x00')
+        fld = struct.pack('<11sc4xBB14x', name, typ, size, deci)
+        f.write(fld)
+
+    # terminator
+    f.write('\r')
+
+    # records
+    for record in records:
+        f.write(' ')                        # deletion flag
+        for (typ, size, deci), value in itertools.izip(fieldspecs, record):
+            if typ == "N":
+                value = str(value).rjust(size, ' ')
+            elif typ == 'D':
+                value = value.strftime('%Y%m%d')
+            elif typ == 'L':
+                value = str(value)[0].upper()
+            else:
+                value = str(value)[:size].ljust(size, ' ')
+            assert len(value) == size
+            f.write(value)
+
+    # End of file
+    f.write('\x1A')

data/misc/shp_normalizer.py

@@ -0,0 +1,58 @@
+from chardet.universaldetector import UniversalDetector
+import os.path
+import sys
+import dbfUtils
+import sys
+from osgeo import osr
+from urllib import urlencode
+from urllib2 import urlopen
+import json
+import subprocess
+
+shp_file = sys.argv[1]
+name = sys.argv[2]
+
+dbf_file = shp_file[0:-4] + '.dbf'
+prj_file = shp_file[0:-4] + '.prj'
+
+
+#Try detecting the SRID, by default we set to 4326 and hope the best
+#srid=4326
+if os.path.isfile(prj_file):
+  prj_filef = open(prj_file, 'r')
+  prj_txt = prj_filef.read()
+  prj_filef.close()
+  srs = osr.SpatialReference()
+  srs.ImportFromESRI([prj_txt])
+  srs.AutoIdentifyEPSG()
+  code = srs.GetAuthorityCode(None)
+  if code:
+    srid = code
+  else:
+    #Ok, no luck, lets try with the OpenGeo service
+    query = urlencode({
+        'exact' : True,
+        'error' : True,
+        'mode' : 'wkt',
+        'terms' : prj_txt})
+    webres = urlopen('http://prj2epsg.org/search.json', query)
+    jres = json.loads(webres.read())
+    if jres['codes']:
+      srid = int(jres['codes'][0]['code'])
+
+#Try to detect the encoding
+dbf = open(dbf_file, 'rb')
+db = dbfUtils.dbfreader(dbf)
+
+detector = UniversalDetector()
+for row in db:
+  detector.feed(str(row))
+  if detector.done: break
+detector.close()
+dbf.close()
+
+encoding = detector.result["encoding"]
+if encoding=="ascii":
+    encoding="LATIN1"
+
+print " -r %s -s 4326 -e -i -I -g the_geom -W %s %s %s" %(srid,encoding,shp_file,name)

data/misc/srid_from_gdal.py

@@ -0,0 +1,11 @@
+import sys
+from osgeo import gdal, osr
+
+f = sys.argv[1]
+g = gdal.Open(f)
+proj = g.GetProjection()
+srs = osr.SpatialReference()
+srs.ImportFromESRI([proj])
+srs.AutoIdentifyEPSG()
+srid = srs.GetAuthorityCode(None)
+print "%s" %(srid)

data/spec/export_spec.rb

@@ -0,0 +1,60 @@
+# coding: UTF-8
+
+require 'spec_helper'
+require File.expand_path("../../lib/cartodb-importer", __FILE__)
+require File.expand_path("../../lib/cartodb-exporter", __FILE__)
+
+describe CartoDB::Exporter do
+  describe "#CSV" do
+    it "should export a ZIP archive containing a CSV file to the /tmp directory named with a unique string" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/clubbing.csv", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      run = importer.import!
+      
+      exporter = CartoDB::Exporter.new :export_to_file => 'clubbing', 
+                                       :type => 'csv',
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+    
+      result = exporter.export!
+      result.name.should == 'clubbing'
+      result.import_type.should == '.csv'
+    end
+  end
+  describe "#KML" do
+    it "should export a KMZ file to the /tmp directory named with a unique string" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/clubbing.csv", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      run = importer.import!
+
+      exporter = CartoDB::Exporter.new :export_to_file => 'clubbing', 
+                                       :type => 'kml',
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+
+      result = exporter.export!
+      result.name.should == 'clubbing'
+      result.import_type.should == '.kml'
+    end
+  end
+  describe "#SHP" do
+    it "should export a SHP file set as a ZIP to the /tmp directory named with a unique string" do
+
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/EjemploVizzuality.zip", __FILE__),
+                                         :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                         :host => 'localhost', :port => 5432
+      run = importer.import!
+
+      exporter = CartoDB::Exporter.new :export_to_file => 'vizzuality_shp', 
+                                       :type => 'shp',
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+
+      result = exporter.export!
+      result.name.should == 'vizzuality_shp'
+      result.import_type.should == '.shp'
+    end
+  end
+end

data/spec/import_spec.rb

@@ -0,0 +1,252 @@
+# coding: UTF-8
+
+require 'spec_helper'
+require File.expand_path("../../lib/cartodb-importer", __FILE__)
+
+describe CartoDB::Importer do
+  it "should raise an error if :import_from_file option is blank" do
+    lambda { 
+      CartoDB::Importer.new 
+    }.should raise_error("import_from_file value can't be nil")
+  end
+  
+  it "should get the name from the options" do
+    importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/clubbing.csv", __FILE__),
+                                     :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                     :host => 'localhost', :port => 5432, :suggested_name => 'prefered_name'
+    result = importer.import!
+    result.name.should == 'prefered_name'
+    result.rows_imported.should == 1998
+    result.import_type.should == '.csv'
+  end
+  
+  it "should remove the table from the database if an exception happens" do
+    options = { :import_from_file => File.expand_path("../support/data/empty.csv", __FILE__),
+                :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                :host => 'localhost', :port => 5432 }
+    importer = CartoDB::Importer.new options
+    lambda { 
+      importer.import!
+    }.should raise_error
+    
+    db_connection = Sequel.connect("postgres://#{options[:username]}:#{options[:password]}@#{options[:host]}:#{options[:port]}/#{options[:database]}")
+    db_connection.tables.should_not include(:empty)
+  end
+  
+  it "should keep existing tables when trying to import a new one with the same name as an existing one and fails" do
+    options = { :import_from_file => File.expand_path("../support/data/empty.csv", __FILE__),
+                :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                :host => 'localhost', :port => 5432, :suggested_name => "testing" }
+                
+    importer = CartoDB::Importer.new(options.merge(:import_from_file => File.expand_path("../support/data/clubbing.csv", __FILE__)))
+    result = importer.import!
+    result.import_type.should == '.csv'
+    
+    importer = CartoDB::Importer.new(options)
+    lambda { 
+      importer.import!
+    }.should raise_error
+
+    db_connection = Sequel.connect("postgres://#{options[:username]}:#{options[:password]}@#{options[:host]}:#{options[:port]}/#{options[:database]}")
+    db_connection.tables.should include(:testing)
+  end
+  
+  it "should suggest a new table name of the format _n if the previous table exists" do
+    importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/clubbing.csv", __FILE__),
+                                     :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                     :host => 'localhost', :port => 5432, :suggested_name => 'prefered_name'
+    result = importer.import!
+    result.name.should == 'prefered_name'
+    result.rows_imported.should == 1998
+    result.import_type.should == '.csv'
+
+    importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/clubbing.csv", __FILE__),
+                                     :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                     :host => 'localhost', :port => 5432, :suggested_name => 'prefered_name'
+    result = importer.import!
+    result.name.should == 'prefered_name_2'
+    result.rows_imported.should == 1998
+    result.import_type.should == '.csv'
+  end
+  
+  it "should sanitize column names" do
+    options = { :import_from_file => File.expand_path("../support/data/twitters.csv", __FILE__),
+                                     :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                     :host => 'localhost', :port => 5432, :suggested_name => 'prefered_name' }
+    importer = CartoDB::Importer.new(options)
+    result = importer.import!
+    result.name.should == 'prefered_name'
+    result.rows_imported.should == 7
+    result.import_type.should == '.csv'
+    
+    db_connection = Sequel.connect("postgres://#{options[:username]}:#{options[:password]}@#{options[:host]}:#{options[:port]}/#{options[:database]}")
+    db_connection.tables.should include(:prefered_name)
+    columns = db_connection.schema(:prefered_name).map{|s| s[0].to_s}
+    expected_columns = ["url","login","country","followers_count"]
+    (columns & expected_columns).sort.should == expected_columns.sort
+  end
+  
+  pending "should escape reserved column names" do
+    options = { :import_from_file => File.expand_path("../support/data/reserved_columns.csv", __FILE__),
+                                     :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                     :host => 'localhost', :port => 5432, :suggested_name => 'prefered_name' }
+    importer = CartoDB::Importer.new(options)
+    result = importer.import!
+    result.name.should == 'prefered_name'
+    result.rows_imported.should == 7
+    result.import_type.should == '.csv'
+    
+    db_connection = Sequel.connect("postgres://#{options[:username]}:#{options[:password]}@#{options[:host]}:#{options[:port]}/#{options[:database]}")
+    db_connection.tables.should include(:prefered_name)
+    columns = db_connection.schema(:prefered_name).map{|s| s[0].to_s}
+    expected_columns = ["url","login","country","followers_count", "_xmin"]
+    (columns & expected_columns).sort.should == expected_columns.sort
+  end
+  
+  describe "#ZIP" do
+    it "should import CSV even from a ZIP file" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/pino.zip", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'data'
+      result.rows_imported.should == 4
+      result.import_type.should == '.csv'
+    end
+
+    it "should import CSV even from a ZIP file with the given name" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/pino.zip", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432, :suggested_name => "table123"
+      result = importer.import!
+      result.name.should == 'table123'
+      result.rows_imported.should == 4
+      result.import_type.should == '.csv'
+    end
+  end
+
+  describe "#CSV" do
+    it "should import a CSV file in the given database in a table named like the file" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/clubbing.csv", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'clubbing'
+      result.rows_imported.should == 1998
+      result.import_type.should == '.csv'
+    end
+    it "should import Food Security Aid Map_projects.csv" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/Food Security Aid Map_projects.csv", __FILE__),
+                                     :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                     :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'food_security_aid_map_projects'
+      result.rows_imported.should == 827
+      result.import_type.should == '.csv'
+    end
+    it "should import world_heritage_list.csv" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/world_heritage_list.csv", __FILE__),
+                                     :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                     :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'world_heritage_list'
+      result.rows_imported.should == 937
+      result.import_type.should == '.csv'
+    end
+    it "should import cp_vizzuality_export.csv" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/cp_vizzuality_export.csv", __FILE__),
+                                     :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                     :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'cp_vizzuality_export'
+      result.rows_imported.should == 19235
+      result.import_type.should == '.csv'
+    end
+    
+    # Not supported by cartodb-importer ~ v0.2.1
+    # File in format different than UTF-8
+    pending "should import estaciones.csv" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/estaciones.csv", __FILE__),
+                                     :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                     :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'estaciones'
+      result.rows_imported.should == 29
+      result.import_type.should == '.csv'
+    end
+    it "should import estaciones2.csv" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/estaciones2.csv", __FILE__),
+                                     :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                     :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'estaciones2'
+      result.rows_imported.should == 30
+      result.import_type.should == '.csv'
+    end
+  end
+  
+  describe "#XLSX" do
+    it "should import a XLSX file in the given database in a table named like the file" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/ngos.xlsx", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'ngos'
+      result.rows_imported.should == 76
+      result.import_type.should == '.xlsx'
+    end
+  end
+  
+  describe "#SHP" do
+    it "should import a SHP file in the given database in a table named like the file" do
+      options = { :import_from_file => File.expand_path("../support/data/EjemploVizzuality.zip", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432 }
+                                       
+      importer = CartoDB::Importer.new(options)
+      result = importer.import!
+      result.name.should == 'vizzuality_shp'
+      result.rows_imported.should == 11
+      result.import_type.should == '.shp'
+      
+      db_connection = Sequel.connect("postgres://#{options[:username]}:#{options[:password]}@#{options[:host]}:#{options[:port]}/#{options[:database]}")
+      db_connection.tables.should include(:vizzuality_shp)
+      columns = db_connection.schema(:vizzuality_shp).map{|s| s[0].to_s}
+      
+      expected_columns = ["gid", "subclass", "x", "y", "length", "area", "angle", "name", 
+        "pid", "lot_navteq", "version_na", "vitesse_sp", "id", "nombrerest", "tipocomida", "the_geom"]
+      (columns & expected_columns).sort.should == expected_columns.sort
+    end
+    
+    it "should import SHP file TM_WORLD_BORDERS_SIMPL-0.3.zip" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/TM_WORLD_BORDERS_SIMPL-0.3.zip", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'tm_world_borders_simpl_0_3_shp'
+      result.rows_imported.should == 246
+      result.import_type.should == '.shp'
+    end
+
+    it "should import SHP file TM_WORLD_BORDERS_SIMPL-0.3.zip but set the given name" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/TM_WORLD_BORDERS_SIMPL-0.3.zip", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432, :suggested_name => 'borders'
+      result = importer.import!
+      result.name.should == 'borders'
+      result.rows_imported.should == 246
+      result.import_type.should == '.shp'
+    end
+  end
+  describe "#GTIFF" do
+    it "should import a GTIFF file in the given database in a table named like the file" do
+      importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/GLOBAL_ELEVATION_SIMPLE.zip", __FILE__),
+                                       :database => "cartodb_importer_test", :username => 'postgres', :password => '',
+                                       :host => 'localhost', :port => 5432
+      result = importer.import!
+      result.name.should == 'global_elevation_simple_tif'
+      result.rows_imported.should == 1500
+      result.import_type.should == '.tif'
+    end
+  end  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+# coding: UTF-8
+
+require 'rubygems'
+require 'bundler'
+Bundler.setup
+require 'sequel'
+
+require File.expand_path("../support/database_connection", __FILE__)
+
+RSpec.configure do |config|
+  config.mock_with :mocha
+  
+  config.before(:each) do
+    CartoDB::DatabaseConnection.connection.tables.each do |t| 
+      next if %W{ raster_columns raster_overviews geography_columns geometry_columns spatial_ref_sys }.include?(t.to_s)
+      CartoDB::DatabaseConnection.connection.drop_table(t)
+    end
+  end
+end