cartodb-importer 0.2.6 → 0.2.7
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/cartodb-importer/importer.rb +4 -3
- data/lib/cartodb-importer/version.rb +1 -1
- data/misc/shp_normalizer.py +16 -14
- data/spec/import_spec.rb +15 -16
- metadata +21 -21
@@ -89,8 +89,9 @@ module CartoDB
|
|
89
89
|
entry.extract("/tmp/#{name}")
|
90
90
|
end
|
91
91
|
end
|
92
|
-
|
92
|
+
|
93
93
|
import_type = @ext
|
94
|
+
|
94
95
|
# These types of files are converted to CSV
|
95
96
|
if %W{ .xls .xlsx .ods }.include?(@ext)
|
96
97
|
new_path = "/tmp/#{@suggested_name}.csv"
|
@@ -146,7 +147,7 @@ module CartoDB
|
|
146
147
|
|
147
148
|
host = @db_configuration[:host] ? "-h #{@db_configuration[:host]}" : ""
|
148
149
|
port = @db_configuration[:port] ? "-p #{@db_configuration[:port]}" : ""
|
149
|
-
|
150
|
+
#@suggested_name = get_valid_name(File.basename(path).tr('.','_').downcase.sanitize) unless @force_name
|
150
151
|
random_table_name = "importing_#{Time.now.to_i}_#{@suggested_name}"
|
151
152
|
|
152
153
|
normalizer_command = "#{python_bin_path} -Wignore #{File.expand_path("../../../misc/shp_normalizer.py", __FILE__)} #{path} #{random_table_name}"
|
@@ -185,7 +186,7 @@ module CartoDB
|
|
185
186
|
|
186
187
|
host = @db_configuration[:host] ? "-h #{@db_configuration[:host]}" : ""
|
187
188
|
port = @db_configuration[:port] ? "-p #{@db_configuration[:port]}" : ""
|
188
|
-
|
189
|
+
#@suggested_name = get_valid_name(File.basename(path).tr('.','_').downcase.sanitize) unless @force_name
|
189
190
|
random_table_name = "importing_#{Time.now.to_i}_#{@suggested_name}"
|
190
191
|
|
191
192
|
gdal_command = "#{python_bin_path} -Wignore #{File.expand_path("../../../misc/srid_from_gdal.py", __FILE__)} #{path}"
|
data/misc/shp_normalizer.py
CHANGED
@@ -39,19 +39,21 @@ if os.path.isfile(prj_file):
|
|
39
39
|
if jres['codes']:
|
40
40
|
srid = int(jres['codes'][0]['code'])
|
41
41
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
detector = UniversalDetector()
|
47
|
-
for row in db:
|
48
|
-
|
49
|
-
|
50
|
-
detector.close()
|
51
|
-
dbf.close()
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
42
|
+
try:
|
43
|
+
#Try to detect the encoding
|
44
|
+
dbf = open(dbf_file, 'rb')
|
45
|
+
db = dbfUtils.dbfreader(dbf)
|
46
|
+
detector = UniversalDetector()
|
47
|
+
for row in db:
|
48
|
+
detector.feed(str(row))
|
49
|
+
if detector.done: break
|
50
|
+
detector.close()
|
51
|
+
dbf.close()
|
52
|
+
encoding = detector.result["encoding"]
|
53
|
+
if encoding=="ascii":
|
54
|
+
encoding="LATIN1"
|
55
|
+
except:
|
56
|
+
#if encoding detection fails, attempt default UTF8
|
57
|
+
encoding = "UTF8"
|
56
58
|
|
57
59
|
print "%s,%s,%s,%s" % (srid,encoding,shp_file,name)
|
data/spec/import_spec.rb
CHANGED
@@ -205,13 +205,13 @@ describe CartoDB::Importer do
|
|
205
205
|
|
206
206
|
importer = CartoDB::Importer.new(options)
|
207
207
|
result = importer.import!
|
208
|
-
result.name.should == '
|
208
|
+
result.name.should == 'vizzuality'
|
209
209
|
result.rows_imported.should == 11
|
210
210
|
result.import_type.should == '.shp'
|
211
211
|
|
212
212
|
db_connection = Sequel.connect("postgres://#{options[:username]}:#{options[:password]}@#{options[:host]}:#{options[:port]}/#{options[:database]}")
|
213
|
-
db_connection.tables.should include(:
|
214
|
-
columns = db_connection.schema(:
|
213
|
+
db_connection.tables.should include(:vizzuality)
|
214
|
+
columns = db_connection.schema(:vizzuality).map{|s| s[0].to_s}
|
215
215
|
|
216
216
|
expected_columns = ["gid", "subclass", "x", "y", "length", "area", "angle", "name",
|
217
217
|
"pid", "lot_navteq", "version_na", "vitesse_sp", "id", "nombrerest", "tipocomida", "the_geom"]
|
@@ -223,7 +223,7 @@ describe CartoDB::Importer do
|
|
223
223
|
:database => "cartodb_importer_test", :username => 'postgres', :password => '',
|
224
224
|
:host => 'localhost', :port => 5432
|
225
225
|
result = importer.import!
|
226
|
-
result.name.should == '
|
226
|
+
result.name.should == 'tm_world_borders_simpl_0_3'
|
227
227
|
result.rows_imported.should == 246
|
228
228
|
result.import_type.should == '.shp'
|
229
229
|
end
|
@@ -244,29 +244,28 @@ describe CartoDB::Importer do
|
|
244
244
|
:database => "cartodb_importer_test", :username => 'postgres', :password => '',
|
245
245
|
:host => 'localhost', :port => 5432
|
246
246
|
result = importer.import!
|
247
|
-
result.name.should == '
|
247
|
+
result.name.should == 'global_elevation_simple'
|
248
248
|
result.rows_imported.should == 1500
|
249
249
|
result.import_type.should == '.tif'
|
250
250
|
end
|
251
251
|
end
|
252
252
|
describe "Extended" do
|
253
|
-
it "should import
|
254
|
-
importer = CartoDB::Importer.new :import_from_file => File.expand_path("
|
253
|
+
it "should import 2 SHP files incrementing the name of the second" do
|
254
|
+
importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/TM_WORLD_BORDERS_SIMPL-0.3.zip", __FILE__),
|
255
255
|
:database => "cartodb_importer_test", :username => 'postgres', :password => '',
|
256
256
|
:host => 'localhost', :port => 5432
|
257
257
|
result = importer.import!
|
258
|
-
result.name.should == '
|
259
|
-
result.rows_imported.should == 4365
|
258
|
+
result.name.should == 'tm_world_borders_simpl_0_3'
|
259
|
+
#result.rows_imported.should == 4365
|
260
260
|
result.import_type.should == '.shp'
|
261
|
-
|
262
|
-
|
263
|
-
importer = CartoDB::Importer.new :import_from_file => File.expand_path("../../../vm-cartodb/extended_tests/cua2005index_utm.zip", __FILE__),
|
261
|
+
|
262
|
+
importer = CartoDB::Importer.new :import_from_file => File.expand_path("../support/data/TM_WORLD_BORDERS_SIMPL-0.3.zip", __FILE__),
|
264
263
|
:database => "cartodb_importer_test", :username => 'postgres', :password => '',
|
265
264
|
:host => 'localhost', :port => 5432
|
266
|
-
|
267
|
-
|
268
|
-
result.rows_imported.should ==
|
269
|
-
|
265
|
+
result2 = importer.import!
|
266
|
+
result2.name.should == 'tm_world_borders_simpl_0_4'
|
267
|
+
#result.rows_imported.should == 4365
|
268
|
+
result2.import_type.should == '.shp'
|
270
269
|
end
|
271
270
|
end
|
272
271
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cartodb-importer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.7
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -15,7 +15,7 @@ date: 2011-09-13 00:00:00.000000000Z
|
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: pg
|
18
|
-
requirement: &
|
18
|
+
requirement: &70204688871100 !ruby/object:Gem::Requirement
|
19
19
|
none: false
|
20
20
|
requirements:
|
21
21
|
- - ~>
|
@@ -23,10 +23,10 @@ dependencies:
|
|
23
23
|
version: '0.11'
|
24
24
|
type: :runtime
|
25
25
|
prerelease: false
|
26
|
-
version_requirements: *
|
26
|
+
version_requirements: *70204688871100
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: sequel
|
29
|
-
requirement: &
|
29
|
+
requirement: &70204688870460 !ruby/object:Gem::Requirement
|
30
30
|
none: false
|
31
31
|
requirements:
|
32
32
|
- - ! '>='
|
@@ -34,10 +34,10 @@ dependencies:
|
|
34
34
|
version: '0'
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
|
-
version_requirements: *
|
37
|
+
version_requirements: *70204688870460
|
38
38
|
- !ruby/object:Gem::Dependency
|
39
39
|
name: roo
|
40
|
-
requirement: &
|
40
|
+
requirement: &70204688869740 !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
42
42
|
requirements:
|
43
43
|
- - ! '>='
|
@@ -45,10 +45,10 @@ dependencies:
|
|
45
45
|
version: '0'
|
46
46
|
type: :runtime
|
47
47
|
prerelease: false
|
48
|
-
version_requirements: *
|
48
|
+
version_requirements: *70204688869740
|
49
49
|
- !ruby/object:Gem::Dependency
|
50
50
|
name: spreadsheet
|
51
|
-
requirement: &
|
51
|
+
requirement: &70204688869160 !ruby/object:Gem::Requirement
|
52
52
|
none: false
|
53
53
|
requirements:
|
54
54
|
- - ! '>='
|
@@ -56,10 +56,10 @@ dependencies:
|
|
56
56
|
version: '0'
|
57
57
|
type: :runtime
|
58
58
|
prerelease: false
|
59
|
-
version_requirements: *
|
59
|
+
version_requirements: *70204688869160
|
60
60
|
- !ruby/object:Gem::Dependency
|
61
61
|
name: google-spreadsheet-ruby
|
62
|
-
requirement: &
|
62
|
+
requirement: &70204688868520 !ruby/object:Gem::Requirement
|
63
63
|
none: false
|
64
64
|
requirements:
|
65
65
|
- - ! '>='
|
@@ -67,10 +67,10 @@ dependencies:
|
|
67
67
|
version: '0'
|
68
68
|
type: :runtime
|
69
69
|
prerelease: false
|
70
|
-
version_requirements: *
|
70
|
+
version_requirements: *70204688868520
|
71
71
|
- !ruby/object:Gem::Dependency
|
72
72
|
name: rubyzip
|
73
|
-
requirement: &
|
73
|
+
requirement: &70204688867880 !ruby/object:Gem::Requirement
|
74
74
|
none: false
|
75
75
|
requirements:
|
76
76
|
- - ! '>='
|
@@ -78,10 +78,10 @@ dependencies:
|
|
78
78
|
version: '0'
|
79
79
|
type: :runtime
|
80
80
|
prerelease: false
|
81
|
-
version_requirements: *
|
81
|
+
version_requirements: *70204688867880
|
82
82
|
- !ruby/object:Gem::Dependency
|
83
83
|
name: builder
|
84
|
-
requirement: &
|
84
|
+
requirement: &70204688867260 !ruby/object:Gem::Requirement
|
85
85
|
none: false
|
86
86
|
requirements:
|
87
87
|
- - ! '>='
|
@@ -89,10 +89,10 @@ dependencies:
|
|
89
89
|
version: '0'
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
|
-
version_requirements: *
|
92
|
+
version_requirements: *70204688867260
|
93
93
|
- !ruby/object:Gem::Dependency
|
94
94
|
name: rspec
|
95
|
-
requirement: &
|
95
|
+
requirement: &70204688866620 !ruby/object:Gem::Requirement
|
96
96
|
none: false
|
97
97
|
requirements:
|
98
98
|
- - ! '>='
|
@@ -100,10 +100,10 @@ dependencies:
|
|
100
100
|
version: '0'
|
101
101
|
type: :development
|
102
102
|
prerelease: false
|
103
|
-
version_requirements: *
|
103
|
+
version_requirements: *70204688866620
|
104
104
|
- !ruby/object:Gem::Dependency
|
105
105
|
name: mocha
|
106
|
-
requirement: &
|
106
|
+
requirement: &70204688865980 !ruby/object:Gem::Requirement
|
107
107
|
none: false
|
108
108
|
requirements:
|
109
109
|
- - ! '>='
|
@@ -111,10 +111,10 @@ dependencies:
|
|
111
111
|
version: '0'
|
112
112
|
type: :development
|
113
113
|
prerelease: false
|
114
|
-
version_requirements: *
|
114
|
+
version_requirements: *70204688865980
|
115
115
|
- !ruby/object:Gem::Dependency
|
116
116
|
name: ruby-debug19
|
117
|
-
requirement: &
|
117
|
+
requirement: &70204688865360 !ruby/object:Gem::Requirement
|
118
118
|
none: false
|
119
119
|
requirements:
|
120
120
|
- - ! '>='
|
@@ -122,7 +122,7 @@ dependencies:
|
|
122
122
|
version: '0'
|
123
123
|
type: :development
|
124
124
|
prerelease: false
|
125
|
-
version_requirements: *
|
125
|
+
version_requirements: *70204688865360
|
126
126
|
description: Import CSV, SHP, and other files with data into a PostgreSQL table
|
127
127
|
email:
|
128
128
|
- andrew@vizzuality.com
|