geocode_records 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG +10 -0
- data/lib/geocode_records/dump_sql_to_csv.rb +1 -1
- data/lib/geocode_records/update_table_from_csv.rb +33 -46
- data/lib/geocode_records/version.rb +1 -1
- data/lib/geocode_records.rb +1 -1
- data/spec/spec_helper.rb +3 -3
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9f99f57199e7a46638cfdbf0e3f234dff58af60f
|
|
4
|
+
data.tar.gz: 934a907fa2ac94a3285b0129eb04d3b0ef4ea4bf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fa195a1637c5b4f6590e5a53bb52f0b67654ea3cfc803328b81982b52061fedb00f295c98dc3a98c9a7780149c40b27d78f3c12b53c0351cffd27e4bf4aac04b
|
|
7
|
+
data.tar.gz: c83a4804cc4aab1f68eac5e10132dae7bb837c601d4ad291bb7ef1400b7a7c5bb3cfa2c45a61db46647099dace86afba2198f367d577f091ba0f04cbf039afa5
|
data/CHANGELOG
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
require 'csv'
|
|
2
|
-
|
|
3
1
|
class GeocodeRecords
|
|
4
2
|
class UpdateTableFromCsv
|
|
5
3
|
CREATE_TABLE_SQL = (<<-SQL).gsub(' ', '').freeze
|
|
@@ -17,32 +15,20 @@ class GeocodeRecords
|
|
|
17
15
|
)
|
|
18
16
|
SQL
|
|
19
17
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
ss_zipcode,
|
|
35
|
-
ss_latitude,
|
|
36
|
-
ss_longitude,
|
|
37
|
-
ss_default_city_name
|
|
38
|
-
)
|
|
39
|
-
WITH
|
|
40
|
-
skip header = 1,
|
|
41
|
-
fields optionally enclosed by '"',
|
|
42
|
-
fields escaped by double-quote,
|
|
43
|
-
fields terminated by ','
|
|
44
|
-
SET client_encoding to 'utf8';
|
|
45
|
-
SQL
|
|
18
|
+
DESIRED_COLUMNS = %w{
|
|
19
|
+
id
|
|
20
|
+
ss_delivery_line_1
|
|
21
|
+
ss_primary_number
|
|
22
|
+
ss_secondary_number
|
|
23
|
+
ss_city_name
|
|
24
|
+
ss_state_abbreviation
|
|
25
|
+
ss_zipcode
|
|
26
|
+
ss_latitude
|
|
27
|
+
ss_longitude
|
|
28
|
+
ss_default_city_name
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
COPY_SQL = "\\copy $TMP_TABLE_NAME (#{DESIRED_COLUMNS.join(',')}) FROM '$PATH' DELIMITER ',' CSV HEADER"
|
|
46
32
|
|
|
47
33
|
UPDATE_TABLE_SQL = (<<-SQL).gsub(' ', '').freeze
|
|
48
34
|
UPDATE $TABLE_NAME AS target
|
|
@@ -79,7 +65,8 @@ class GeocodeRecords
|
|
|
79
65
|
return unless File.size(path) > 32
|
|
80
66
|
tmp_table_name = create_tmp_table
|
|
81
67
|
begin
|
|
82
|
-
|
|
68
|
+
tmp_csv_path = strip_csv
|
|
69
|
+
load_csv_into_tmp_table path: tmp_csv_path, table_name: tmp_table_name
|
|
83
70
|
update_original_table tmp_table_name
|
|
84
71
|
ensure
|
|
85
72
|
delete_tmp_table tmp_table_name
|
|
@@ -88,43 +75,43 @@ class GeocodeRecords
|
|
|
88
75
|
|
|
89
76
|
def create_tmp_table
|
|
90
77
|
memo = "geocode_records_#{table_name}_#{rand(999999)}".gsub(/[^a-z0-9_]/i, '')
|
|
91
|
-
GeocodeRecords.
|
|
78
|
+
GeocodeRecords.psql(
|
|
92
79
|
database_url,
|
|
93
80
|
CREATE_TABLE_SQL.sub('$TMP_TABLE_NAME', memo)
|
|
94
81
|
)
|
|
95
82
|
memo
|
|
96
83
|
end
|
|
97
84
|
|
|
98
|
-
def
|
|
99
|
-
|
|
100
|
-
File.open(pg_loader_config_path, 'w') { |f| f.write PGLOADER_CONFIG.sub('$INPUT_COLUMNS', input_columns.join(',')).sub('$DATABASE_URL', database_url).sub('$TMP_TABLE_NAME', tmp_table_name).sub('$PATH', path) }
|
|
85
|
+
def strip_csv
|
|
86
|
+
memo = GeocodeRecords.new_tmp_path('stripped')
|
|
101
87
|
GeocodeRecords.system(
|
|
102
|
-
'
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
88
|
+
'xsv',
|
|
89
|
+
'select', DESIRED_COLUMNS.join(','),
|
|
90
|
+
path,
|
|
91
|
+
out: memo
|
|
92
|
+
)
|
|
93
|
+
memo
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def load_csv_into_tmp_table(path:, table_name:)
|
|
97
|
+
GeocodeRecords.psql(
|
|
98
|
+
database_url,
|
|
99
|
+
COPY_SQL.sub('$TMP_TABLE_NAME', table_name).sub('$PATH', path)
|
|
106
100
|
)
|
|
107
|
-
File.unlink pg_loader_config_path
|
|
108
101
|
end
|
|
109
102
|
|
|
110
103
|
def update_original_table(tmp_table_name)
|
|
111
|
-
GeocodeRecords.
|
|
104
|
+
GeocodeRecords.psql(
|
|
112
105
|
database_url,
|
|
113
106
|
UPDATE_TABLE_SQL.sub('$TMP_TABLE_NAME', tmp_table_name).sub('$TABLE_NAME', table_name)
|
|
114
107
|
)
|
|
115
108
|
end
|
|
116
109
|
|
|
117
110
|
def delete_tmp_table(tmp_table_name)
|
|
118
|
-
GeocodeRecords.
|
|
111
|
+
GeocodeRecords.psql(
|
|
119
112
|
database_url,
|
|
120
113
|
"DROP TABLE IF EXISTS #{tmp_table_name}"
|
|
121
114
|
)
|
|
122
115
|
end
|
|
123
|
-
|
|
124
|
-
def input_columns
|
|
125
|
-
CSV.parse_line(File.open(path) { |f| f.gets }).map do |col|
|
|
126
|
-
"#{col} [NULL IF BLANKS]"
|
|
127
|
-
end
|
|
128
|
-
end
|
|
129
116
|
end
|
|
130
117
|
end
|
data/lib/geocode_records.rb
CHANGED
data/spec/spec_helper.rb
CHANGED
|
@@ -8,11 +8,11 @@ ENV['DATABASE_URL'] = "postgresql://127.0.0.1:#{ENV['PGPORT'] || 5432}/#{dbname}
|
|
|
8
8
|
|
|
9
9
|
unless ENV['FAST'] == 'true'
|
|
10
10
|
GeocodeRecords.system('createdb', ENV.fetch('DATABASE_URL')) rescue nil
|
|
11
|
-
GeocodeRecords.
|
|
11
|
+
GeocodeRecords.psql(
|
|
12
12
|
ENV.fetch('DATABASE_URL'),
|
|
13
13
|
'CREATE EXTENSION IF NOT EXISTS postgis'
|
|
14
14
|
)
|
|
15
|
-
GeocodeRecords.
|
|
15
|
+
GeocodeRecords.psql(
|
|
16
16
|
ENV.fetch('DATABASE_URL'),
|
|
17
17
|
'DROP TABLE IF EXISTS homes'
|
|
18
18
|
)
|
|
@@ -33,7 +33,7 @@ unless ENV['FAST'] == 'true'
|
|
|
33
33
|
foo text
|
|
34
34
|
)
|
|
35
35
|
SQL
|
|
36
|
-
GeocodeRecords.
|
|
36
|
+
GeocodeRecords.psql(
|
|
37
37
|
ENV.fetch('DATABASE_URL'),
|
|
38
38
|
sql
|
|
39
39
|
)
|