geocode_records 1.0.1 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG +10 -0
- data/lib/geocode_records/dump_sql_to_csv.rb +1 -1
- data/lib/geocode_records/update_table_from_csv.rb +33 -46
- data/lib/geocode_records/version.rb +1 -1
- data/lib/geocode_records.rb +1 -1
- data/spec/spec_helper.rb +3 -3
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9f99f57199e7a46638cfdbf0e3f234dff58af60f
|
|
4
|
+
data.tar.gz: 934a907fa2ac94a3285b0129eb04d3b0ef4ea4bf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fa195a1637c5b4f6590e5a53bb52f0b67654ea3cfc803328b81982b52061fedb00f295c98dc3a98c9a7780149c40b27d78f3c12b53c0351cffd27e4bf4aac04b
|
|
7
|
+
data.tar.gz: c83a4804cc4aab1f68eac5e10132dae7bb837c601d4ad291bb7ef1400b7a7c5bb3cfa2c45a61db46647099dace86afba2198f367d577f091ba0f04cbf039afa5
|
data/CHANGELOG
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
require 'csv'
|
|
2
|
-
|
|
3
1
|
class GeocodeRecords
|
|
4
2
|
class UpdateTableFromCsv
|
|
5
3
|
CREATE_TABLE_SQL = (<<-SQL).gsub(' ', '').freeze
|
|
@@ -17,32 +15,20 @@ class GeocodeRecords
|
|
|
17
15
|
)
|
|
18
16
|
SQL
|
|
19
17
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
ss_zipcode,
|
|
35
|
-
ss_latitude,
|
|
36
|
-
ss_longitude,
|
|
37
|
-
ss_default_city_name
|
|
38
|
-
)
|
|
39
|
-
WITH
|
|
40
|
-
skip header = 1,
|
|
41
|
-
fields optionally enclosed by '"',
|
|
42
|
-
fields escaped by double-quote,
|
|
43
|
-
fields terminated by ','
|
|
44
|
-
SET client_encoding to 'utf8';
|
|
45
|
-
SQL
|
|
18
|
+
DESIRED_COLUMNS = %w{
|
|
19
|
+
id
|
|
20
|
+
ss_delivery_line_1
|
|
21
|
+
ss_primary_number
|
|
22
|
+
ss_secondary_number
|
|
23
|
+
ss_city_name
|
|
24
|
+
ss_state_abbreviation
|
|
25
|
+
ss_zipcode
|
|
26
|
+
ss_latitude
|
|
27
|
+
ss_longitude
|
|
28
|
+
ss_default_city_name
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
COPY_SQL = "\\copy $TMP_TABLE_NAME (#{DESIRED_COLUMNS.join(',')}) FROM '$PATH' DELIMITER ',' CSV HEADER"
|
|
46
32
|
|
|
47
33
|
UPDATE_TABLE_SQL = (<<-SQL).gsub(' ', '').freeze
|
|
48
34
|
UPDATE $TABLE_NAME AS target
|
|
@@ -79,7 +65,8 @@ class GeocodeRecords
|
|
|
79
65
|
return unless File.size(path) > 32
|
|
80
66
|
tmp_table_name = create_tmp_table
|
|
81
67
|
begin
|
|
82
|
-
|
|
68
|
+
tmp_csv_path = strip_csv
|
|
69
|
+
load_csv_into_tmp_table path: tmp_csv_path, table_name: tmp_table_name
|
|
83
70
|
update_original_table tmp_table_name
|
|
84
71
|
ensure
|
|
85
72
|
delete_tmp_table tmp_table_name
|
|
@@ -88,43 +75,43 @@ class GeocodeRecords
|
|
|
88
75
|
|
|
89
76
|
def create_tmp_table
|
|
90
77
|
memo = "geocode_records_#{table_name}_#{rand(999999)}".gsub(/[^a-z0-9_]/i, '')
|
|
91
|
-
GeocodeRecords.
|
|
78
|
+
GeocodeRecords.psql(
|
|
92
79
|
database_url,
|
|
93
80
|
CREATE_TABLE_SQL.sub('$TMP_TABLE_NAME', memo)
|
|
94
81
|
)
|
|
95
82
|
memo
|
|
96
83
|
end
|
|
97
84
|
|
|
98
|
-
def
|
|
99
|
-
|
|
100
|
-
File.open(pg_loader_config_path, 'w') { |f| f.write PGLOADER_CONFIG.sub('$INPUT_COLUMNS', input_columns.join(',')).sub('$DATABASE_URL', database_url).sub('$TMP_TABLE_NAME', tmp_table_name).sub('$PATH', path) }
|
|
85
|
+
def strip_csv
|
|
86
|
+
memo = GeocodeRecords.new_tmp_path('stripped')
|
|
101
87
|
GeocodeRecords.system(
|
|
102
|
-
'
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
88
|
+
'xsv',
|
|
89
|
+
'select', DESIRED_COLUMNS.join(','),
|
|
90
|
+
path,
|
|
91
|
+
out: memo
|
|
92
|
+
)
|
|
93
|
+
memo
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def load_csv_into_tmp_table(path:, table_name:)
|
|
97
|
+
GeocodeRecords.psql(
|
|
98
|
+
database_url,
|
|
99
|
+
COPY_SQL.sub('$TMP_TABLE_NAME', table_name).sub('$PATH', path)
|
|
106
100
|
)
|
|
107
|
-
File.unlink pg_loader_config_path
|
|
108
101
|
end
|
|
109
102
|
|
|
110
103
|
def update_original_table(tmp_table_name)
|
|
111
|
-
GeocodeRecords.
|
|
104
|
+
GeocodeRecords.psql(
|
|
112
105
|
database_url,
|
|
113
106
|
UPDATE_TABLE_SQL.sub('$TMP_TABLE_NAME', tmp_table_name).sub('$TABLE_NAME', table_name)
|
|
114
107
|
)
|
|
115
108
|
end
|
|
116
109
|
|
|
117
110
|
def delete_tmp_table(tmp_table_name)
|
|
118
|
-
GeocodeRecords.
|
|
111
|
+
GeocodeRecords.psql(
|
|
119
112
|
database_url,
|
|
120
113
|
"DROP TABLE IF EXISTS #{tmp_table_name}"
|
|
121
114
|
)
|
|
122
115
|
end
|
|
123
|
-
|
|
124
|
-
def input_columns
|
|
125
|
-
CSV.parse_line(File.open(path) { |f| f.gets }).map do |col|
|
|
126
|
-
"#{col} [NULL IF BLANKS]"
|
|
127
|
-
end
|
|
128
|
-
end
|
|
129
116
|
end
|
|
130
117
|
end
|
data/lib/geocode_records.rb
CHANGED
data/spec/spec_helper.rb
CHANGED
|
@@ -8,11 +8,11 @@ ENV['DATABASE_URL'] = "postgresql://127.0.0.1:#{ENV['PGPORT'] || 5432}/#{dbname}
|
|
|
8
8
|
|
|
9
9
|
unless ENV['FAST'] == 'true'
|
|
10
10
|
GeocodeRecords.system('createdb', ENV.fetch('DATABASE_URL')) rescue nil
|
|
11
|
-
GeocodeRecords.
|
|
11
|
+
GeocodeRecords.psql(
|
|
12
12
|
ENV.fetch('DATABASE_URL'),
|
|
13
13
|
'CREATE EXTENSION IF NOT EXISTS postgis'
|
|
14
14
|
)
|
|
15
|
-
GeocodeRecords.
|
|
15
|
+
GeocodeRecords.psql(
|
|
16
16
|
ENV.fetch('DATABASE_URL'),
|
|
17
17
|
'DROP TABLE IF EXISTS homes'
|
|
18
18
|
)
|
|
@@ -33,7 +33,7 @@ unless ENV['FAST'] == 'true'
|
|
|
33
33
|
foo text
|
|
34
34
|
)
|
|
35
35
|
SQL
|
|
36
|
-
GeocodeRecords.
|
|
36
|
+
GeocodeRecords.psql(
|
|
37
37
|
ENV.fetch('DATABASE_URL'),
|
|
38
38
|
sql
|
|
39
39
|
)
|