geocode_records 1.0.1 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 83fb253418d4002fbfc73f142af3b3727941757c
4
- data.tar.gz: 8b62d3741a7aeec0386a195075b5a71b0ed30ddb
3
+ metadata.gz: 9f99f57199e7a46638cfdbf0e3f234dff58af60f
4
+ data.tar.gz: 934a907fa2ac94a3285b0129eb04d3b0ef4ea4bf
5
5
  SHA512:
6
- metadata.gz: 284069b053dc0a918487f3089bec1b73c54cc05f252480ca9701f47e654eacc5852c993736ed3f4bb22d86353acd11043f7f7b79671ec9185cd203a9a3f0602d
7
- data.tar.gz: 735420c63e805174ec2dd5e4c4f18cf3b07c6c7d74455e5b52a7a4354a25485912639838984d520686100028bd2b8ab50c508da9e83c428bf8b4c142295c0877
6
+ metadata.gz: fa195a1637c5b4f6590e5a53bb52f0b67654ea3cfc803328b81982b52061fedb00f295c98dc3a98c9a7780149c40b27d78f3c12b53c0351cffd27e4bf4aac04b
7
+ data.tar.gz: c83a4804cc4aab1f68eac5e10132dae7bb837c601d4ad291bb7ef1400b7a7c5bb3cfa2c45a61db46647099dace86afba2198f367d577f091ba0f04cbf039afa5
data/CHANGELOG CHANGED
@@ -1,3 +1,13 @@
1
+ 1.1.0 / 2018-01-16
2
+
3
+ * Breaking changes
4
+
5
+ * Require xsv
6
+
7
+ * Enhancements
8
+
9
+ * Remove pgloader, which was hard to build on alpine
10
+
1
11
  1.0.1 / 2018-01-16
2
12
 
3
13
  * Enhancements
@@ -18,7 +18,7 @@ class GeocodeRecords
18
18
 
19
19
  def perform
20
20
  memo = GeocodeRecords.new_tmp_path(subquery || table_name)
21
- GeocodeRecords.run_sql(
21
+ GeocodeRecords.psql(
22
22
  database_url,
23
23
  "\\copy (#{sql}) TO '#{memo}' DELIMITER ',' CSV HEADER"
24
24
  )
@@ -1,5 +1,3 @@
1
- require 'csv'
2
-
3
1
  class GeocodeRecords
4
2
  class UpdateTableFromCsv
5
3
  CREATE_TABLE_SQL = (<<-SQL).gsub(' ', '').freeze
@@ -17,32 +15,20 @@ class GeocodeRecords
17
15
  )
18
16
  SQL
19
17
 
20
- PGLOADER_CONFIG = <<-SQL
21
- LOAD CSV
22
- FROM '$PATH'
23
- (
24
- $INPUT_COLUMNS
25
- )
26
- INTO $DATABASE_URL?$TMP_TABLE_NAME
27
- (
28
- id,
29
- ss_delivery_line_1,
30
- ss_primary_number,
31
- ss_secondary_number,
32
- ss_city_name,
33
- ss_state_abbreviation,
34
- ss_zipcode,
35
- ss_latitude,
36
- ss_longitude,
37
- ss_default_city_name
38
- )
39
- WITH
40
- skip header = 1,
41
- fields optionally enclosed by '"',
42
- fields escaped by double-quote,
43
- fields terminated by ','
44
- SET client_encoding to 'utf8';
45
- SQL
18
+ DESIRED_COLUMNS = %w{
19
+ id
20
+ ss_delivery_line_1
21
+ ss_primary_number
22
+ ss_secondary_number
23
+ ss_city_name
24
+ ss_state_abbreviation
25
+ ss_zipcode
26
+ ss_latitude
27
+ ss_longitude
28
+ ss_default_city_name
29
+ }
30
+
31
+ COPY_SQL = "\\copy $TMP_TABLE_NAME (#{DESIRED_COLUMNS.join(',')}) FROM '$PATH' DELIMITER ',' CSV HEADER"
46
32
 
47
33
  UPDATE_TABLE_SQL = (<<-SQL).gsub(' ', '').freeze
48
34
  UPDATE $TABLE_NAME AS target
@@ -79,7 +65,8 @@ class GeocodeRecords
79
65
  return unless File.size(path) > 32
80
66
  tmp_table_name = create_tmp_table
81
67
  begin
82
- load_csv_into_tmp_table tmp_table_name
68
+ tmp_csv_path = strip_csv
69
+ load_csv_into_tmp_table path: tmp_csv_path, table_name: tmp_table_name
83
70
  update_original_table tmp_table_name
84
71
  ensure
85
72
  delete_tmp_table tmp_table_name
@@ -88,43 +75,43 @@ class GeocodeRecords
88
75
 
89
76
  def create_tmp_table
90
77
  memo = "geocode_records_#{table_name}_#{rand(999999)}".gsub(/[^a-z0-9_]/i, '')
91
- GeocodeRecords.run_sql(
78
+ GeocodeRecords.psql(
92
79
  database_url,
93
80
  CREATE_TABLE_SQL.sub('$TMP_TABLE_NAME', memo)
94
81
  )
95
82
  memo
96
83
  end
97
84
 
98
- def load_csv_into_tmp_table(tmp_table_name)
99
- pg_loader_config_path = GeocodeRecords.new_tmp_path('pgloader')
100
- File.open(pg_loader_config_path, 'w') { |f| f.write PGLOADER_CONFIG.sub('$INPUT_COLUMNS', input_columns.join(',')).sub('$DATABASE_URL', database_url).sub('$TMP_TABLE_NAME', tmp_table_name).sub('$PATH', path) }
85
+ def strip_csv
86
+ memo = GeocodeRecords.new_tmp_path('stripped')
101
87
  GeocodeRecords.system(
102
- 'pgloader',
103
- # '--debug',
104
- '--quiet',
105
- pg_loader_config_path
88
+ 'xsv',
89
+ 'select', DESIRED_COLUMNS.join(','),
90
+ path,
91
+ out: memo
92
+ )
93
+ memo
94
+ end
95
+
96
+ def load_csv_into_tmp_table(path:, table_name:)
97
+ GeocodeRecords.psql(
98
+ database_url,
99
+ COPY_SQL.sub('$TMP_TABLE_NAME', table_name).sub('$PATH', path)
106
100
  )
107
- File.unlink pg_loader_config_path
108
101
  end
109
102
 
110
103
  def update_original_table(tmp_table_name)
111
- GeocodeRecords.run_sql(
104
+ GeocodeRecords.psql(
112
105
  database_url,
113
106
  UPDATE_TABLE_SQL.sub('$TMP_TABLE_NAME', tmp_table_name).sub('$TABLE_NAME', table_name)
114
107
  )
115
108
  end
116
109
 
117
110
  def delete_tmp_table(tmp_table_name)
118
- GeocodeRecords.run_sql(
111
+ GeocodeRecords.psql(
119
112
  database_url,
120
113
  "DROP TABLE IF EXISTS #{tmp_table_name}"
121
114
  )
122
115
  end
123
-
124
- def input_columns
125
- CSV.parse_line(File.open(path) { |f| f.gets }).map do |col|
126
- "#{col} [NULL IF BLANKS]"
127
- end
128
- end
129
116
  end
130
117
  end
@@ -1,3 +1,3 @@
1
1
  class GeocodeRecords
2
- VERSION = '1.0.1'
2
+ VERSION = '1.1.0'
3
3
  end
@@ -23,7 +23,7 @@ class GeocodeRecords
23
23
  nil
24
24
  end
25
25
 
26
- def run_sql(database_url, sql)
26
+ def psql(database_url, sql)
27
27
  system(
28
28
  'psql',
29
29
  database_url,
data/spec/spec_helper.rb CHANGED
@@ -8,11 +8,11 @@ ENV['DATABASE_URL'] = "postgresql://127.0.0.1:#{ENV['PGPORT'] || 5432}/#{dbname}
8
8
 
9
9
  unless ENV['FAST'] == 'true'
10
10
  GeocodeRecords.system('createdb', ENV.fetch('DATABASE_URL')) rescue nil
11
- GeocodeRecords.run_sql(
11
+ GeocodeRecords.psql(
12
12
  ENV.fetch('DATABASE_URL'),
13
13
  'CREATE EXTENSION IF NOT EXISTS postgis'
14
14
  )
15
- GeocodeRecords.run_sql(
15
+ GeocodeRecords.psql(
16
16
  ENV.fetch('DATABASE_URL'),
17
17
  'DROP TABLE IF EXISTS homes'
18
18
  )
@@ -33,7 +33,7 @@ unless ENV['FAST'] == 'true'
33
33
  foo text
34
34
  )
35
35
  SQL
36
- GeocodeRecords.run_sql(
36
+ GeocodeRecords.psql(
37
37
  ENV.fetch('DATABASE_URL'),
38
38
  sql
39
39
  )
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: geocode_records
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.1
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Seamus Abshere