ofac 1.2.1 → 1.2.2
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +5 -0
- data/VERSION.yml +1 -1
- data/lib/ofac/models/ofac_sdn_loader.rb +76 -64
- data/ofac.gemspec +2 -2
- data/rdoc/classes/OfacSdnLoader.html +6 -3
- data/rdoc/created.rid +1 -1
- data/rdoc/files/lib/ofac/models/ofac_sdn_loader_rb.html +1 -1
- data/rdoc/index.html +2 -2
- metadata +3 -3
data/History.txt
CHANGED
data/VERSION.yml
CHANGED
@@ -9,12 +9,14 @@ class OfacSdnLoader
|
|
9
9
|
def self.load_current_sdn_file
|
10
10
|
puts "Reloading OFAC sdn data"
|
11
11
|
puts "Downloading OFAC data from http://www.treas.gov/offices/enforcement/ofac/sdn"
|
12
|
+
yield "Downloading OFAC data from http://www.treas.gov/offices/enforcement/ofac/sdn" if block_given?
|
12
13
|
#get the 3 data files
|
13
|
-
sdn
|
14
|
+
sdn = Tempfile.new('sdn')
|
14
15
|
bytes = sdn.write(Net::HTTP.get(URI.parse('http://www.treasury.gov/ofac/downloads/sdn.pip')))
|
15
16
|
sdn.rewind
|
16
17
|
if bytes == 0 || convert_line_to_array(sdn.readline).size != 12
|
17
18
|
puts "Trouble downloading file. The url may have changed."
|
19
|
+
yield "Trouble downloading file. The url may have changed." if block_given?
|
18
20
|
return
|
19
21
|
else
|
20
22
|
sdn.rewind
|
@@ -25,9 +27,10 @@ class OfacSdnLoader
|
|
25
27
|
alt = Tempfile.new('sdn')
|
26
28
|
alt.write(Net::HTTP.get(URI.parse('http://www.treasury.gov/ofac/downloads/alt.pip')))
|
27
29
|
alt.rewind
|
28
|
-
|
30
|
+
|
29
31
|
if OfacSdn.connection.kind_of?(ActiveRecord::ConnectionAdapters::MysqlAdapter)
|
30
32
|
puts "Converting file to csv format for Mysql import. This could take several minutes."
|
33
|
+
yield "Converting file to csv format for Mysql import. This could take several minutes." if block_given?
|
31
34
|
|
32
35
|
csv_file = convert_to_flattened_csv(sdn, address, alt)
|
33
36
|
|
@@ -47,9 +50,9 @@ class OfacSdnLoader
|
|
47
50
|
#convert the file's null value to an empty string
|
48
51
|
#and removes " chars.
|
49
52
|
def self.clean_file_string(line)
|
50
|
-
line.gsub!(/-0-(\s)?/,'')
|
51
|
-
line.gsub!(/\n/,'')
|
52
|
-
line.gsub(/\"/,'')
|
53
|
+
line.gsub!(/-0-(\s)?/, '')
|
54
|
+
line.gsub!(/\n/, '')
|
55
|
+
line.gsub(/\"/, '')
|
53
56
|
end
|
54
57
|
|
55
58
|
#split the line into an array
|
@@ -61,7 +64,7 @@ class OfacSdnLoader
|
|
61
64
|
#1 array of address records and one array of alt records
|
62
65
|
def self.foreign_key_records(sdn_id)
|
63
66
|
address_records = []
|
64
|
-
alt_records
|
67
|
+
alt_records = []
|
65
68
|
|
66
69
|
#the first element in each array is the primary and foreign keys
|
67
70
|
#we are denormalizing the data
|
@@ -94,18 +97,18 @@ class OfacSdnLoader
|
|
94
97
|
def self.sdn_text_to_hash(line)
|
95
98
|
unless line.nil?
|
96
99
|
value_array = convert_line_to_array(line)
|
97
|
-
{:id
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
100
|
+
{:id => value_array[0],
|
101
|
+
:name => value_array[1],
|
102
|
+
:sdn_type => value_array[2],
|
103
|
+
:program => value_array[3],
|
104
|
+
:title => value_array[4],
|
105
|
+
:vessel_call_sign => value_array[5],
|
106
|
+
:vessel_type => value_array[6],
|
107
|
+
:vessel_tonnage => value_array[7],
|
108
|
+
:gross_registered_tonnage => value_array[8],
|
109
|
+
:vessel_flag => value_array[9],
|
110
|
+
:vessel_owner => value_array[10],
|
111
|
+
:remarks => value_array[11]
|
109
112
|
}
|
110
113
|
end
|
111
114
|
end
|
@@ -113,10 +116,10 @@ class OfacSdnLoader
|
|
113
116
|
def self.address_text_to_hash(line)
|
114
117
|
unless line.nil?
|
115
118
|
value_array = convert_line_to_array(line)
|
116
|
-
{:id
|
117
|
-
:address
|
118
|
-
:city
|
119
|
-
:country
|
119
|
+
{:id => value_array[0],
|
120
|
+
:address => value_array[2],
|
121
|
+
:city => value_array[3],
|
122
|
+
:country => value_array[4],
|
120
123
|
:address_remarks => value_array[5]
|
121
124
|
}
|
122
125
|
end
|
@@ -125,9 +128,9 @@ class OfacSdnLoader
|
|
125
128
|
def self.alt_text_to_hash(line)
|
126
129
|
unless line.nil?
|
127
130
|
value_array = convert_line_to_array(line)
|
128
|
-
{:id
|
129
|
-
:alternate_identity_type
|
130
|
-
:alternate_identity_name
|
131
|
+
{:id => value_array[0],
|
132
|
+
:alternate_identity_type => value_array[2],
|
133
|
+
:alternate_identity_name => value_array[3],
|
131
134
|
:alternate_identity_remarks => value_array[4]
|
132
135
|
}
|
133
136
|
end
|
@@ -136,68 +139,68 @@ class OfacSdnLoader
|
|
136
139
|
def self.convert_hash_to_mysql_import_string(record_hash)
|
137
140
|
# empty field for id to be generated by mysql.
|
138
141
|
new_line = "``|" +
|
139
|
-
|
142
|
+
# :name
|
140
143
|
"`#{record_hash[:name]}`|" +
|
141
|
-
|
144
|
+
# :sdn_type
|
142
145
|
"`#{record_hash[:sdn_type]}`|" +
|
143
|
-
|
146
|
+
# :program
|
144
147
|
"`#{record_hash[:program]}`|" +
|
145
|
-
|
148
|
+
# :title
|
146
149
|
"`#{record_hash[:title]}`|" +
|
147
|
-
|
150
|
+
# :vessel_call_sign
|
148
151
|
"`#{record_hash[:vessel_call_sign]}`|" +
|
149
|
-
|
152
|
+
# :vessel_type
|
150
153
|
"`#{record_hash[:vessel_type]}`|" +
|
151
|
-
|
154
|
+
# :vessel_tonnage
|
152
155
|
"`#{record_hash[:vessel_tonnage]}`|" +
|
153
|
-
|
156
|
+
# :gross_registered_tonnage
|
154
157
|
"`#{record_hash[:gross_registered_tonnage]}`|" +
|
155
|
-
|
158
|
+
# :vessel_flag
|
156
159
|
"`#{record_hash[:vessel_flag]}`|" +
|
157
|
-
|
160
|
+
# :vessel_owner
|
158
161
|
"`#{record_hash[:vessel_owner]}`|" +
|
159
|
-
|
162
|
+
# :remarks
|
160
163
|
"`#{record_hash[:remarks]}`|" +
|
161
|
-
|
164
|
+
# :address
|
162
165
|
"`#{record_hash[:address]}`|" +
|
163
|
-
|
166
|
+
# :city
|
164
167
|
"`#{record_hash[:city]}`|" +
|
165
|
-
|
168
|
+
# :country
|
166
169
|
"`#{record_hash[:country]}`|" +
|
167
|
-
|
170
|
+
# :address_remarks
|
168
171
|
"`#{record_hash[:address_remarks]}`|" +
|
169
|
-
|
172
|
+
# :alternate_identity_type
|
170
173
|
"`#{record_hash[:alternate_identity_type]}`|" +
|
171
|
-
|
174
|
+
# :alternate_identity_name
|
172
175
|
"`#{record_hash[:alternate_identity_name]}`|" +
|
173
|
-
|
176
|
+
# :alternate_identity_remarks
|
174
177
|
"`#{record_hash[:alternate_identity_remarks]}`|" +
|
175
|
-
|
178
|
+
#:created_at
|
176
179
|
"`#{Time.now.to_s(:db)}`|" +
|
177
|
-
|
180
|
+
# updated_at
|
178
181
|
"`#{Time.now.to_s(:db)}`" + "\n"
|
179
182
|
|
180
183
|
new_line
|
181
184
|
end
|
182
185
|
|
183
186
|
def self.convert_to_flattened_csv(sdn_file, address_file, alt_file)
|
184
|
-
@address
|
185
|
-
@alt
|
187
|
+
@address = address_file
|
188
|
+
@alt = alt_file
|
186
189
|
|
187
|
-
csv_file
|
190
|
+
csv_file = Tempfile.new("ofac") # create temp file for converted csv format.
|
188
191
|
#get the first line from the address and alt files
|
189
192
|
@current_address_hash = address_text_to_hash(@address.gets)
|
190
|
-
@current_alt_hash
|
193
|
+
@current_alt_hash = alt_text_to_hash(@alt.gets)
|
194
|
+
|
195
|
+
start = Time.now
|
191
196
|
|
192
|
-
start = Time.now
|
193
|
-
|
194
197
|
sdn_file.each_with_index do |line, i|
|
195
|
-
|
198
|
+
|
196
199
|
#initialize the address and alt atributes to empty strings
|
197
200
|
address_attributes = address_text_to_hash("|||||")
|
198
|
-
alt_attributes
|
201
|
+
alt_attributes = alt_text_to_hash("||||")
|
199
202
|
|
200
|
-
sdn_attributes
|
203
|
+
sdn_attributes = sdn_text_to_hash(line)
|
201
204
|
|
202
205
|
#get the foreign key records for this sdn
|
203
206
|
address_records, alt_records = foreign_key_records(sdn_attributes[:id])
|
@@ -224,31 +227,35 @@ class OfacSdnLoader
|
|
224
227
|
end
|
225
228
|
end
|
226
229
|
end
|
227
|
-
|
230
|
+
if (i % 1000 == 0) && (i > 0)
|
231
|
+
puts "#{i} records processed."
|
232
|
+
yield "#{i} records processed." if block_given?
|
233
|
+
end
|
228
234
|
end
|
229
235
|
puts "File conversion ran for #{(Time.now - start) / 60} minutes."
|
236
|
+
yield "File conversion ran for #{(Time.now - start) / 60} minutes." if block_given?
|
230
237
|
return csv_file
|
231
238
|
end
|
232
239
|
|
233
240
|
def self.active_record_file_load(sdn_file, address_file, alt_file)
|
234
241
|
@address = address_file
|
235
|
-
@alt
|
242
|
+
@alt = alt_file
|
236
243
|
|
237
244
|
#OFAC data is a complete list, so we have to dump and load
|
238
245
|
OfacSdn.delete_all
|
239
246
|
|
240
247
|
#get the first line from the address and alt files
|
241
248
|
@current_address_hash = address_text_to_hash(@address.gets)
|
242
|
-
@current_alt_hash
|
243
|
-
attributes
|
249
|
+
@current_alt_hash = alt_text_to_hash(@alt.gets)
|
250
|
+
attributes = {}
|
244
251
|
sdn_file.each_with_index do |line, i|
|
245
252
|
|
246
253
|
#initialize the address and alt atributes to empty strings
|
247
254
|
address_attributes = address_text_to_hash("|||||")
|
248
|
-
alt_attributes
|
255
|
+
alt_attributes = alt_text_to_hash("||||")
|
256
|
+
|
257
|
+
sdn_attributes = sdn_text_to_hash(line)
|
249
258
|
|
250
|
-
sdn_attributes = sdn_text_to_hash(line)
|
251
|
-
|
252
259
|
#get the foreign key records for this sdn
|
253
260
|
address_records, alt_records = foreign_key_records(sdn_attributes[:id])
|
254
261
|
|
@@ -282,8 +289,10 @@ class OfacSdnLoader
|
|
282
289
|
end
|
283
290
|
end
|
284
291
|
end
|
285
|
-
|
286
|
-
|
292
|
+
if (i % 5000 == 0) && (i > 0)
|
293
|
+
puts "#{i} records processed."
|
294
|
+
yield "#{i} records processed." if block_given?
|
295
|
+
end
|
287
296
|
end
|
288
297
|
end
|
289
298
|
|
@@ -293,18 +302,21 @@ class OfacSdnLoader
|
|
293
302
|
# see http://dev.mysql.com/doc/refman/5.1/en/load-data.html
|
294
303
|
def self.bulk_mysql_update(csv_file)
|
295
304
|
puts "Deleting all records in ofac_sdn..."
|
305
|
+
yield "Deleting all records in ofac_sdn..." if block_given?
|
296
306
|
|
297
307
|
#OFAC data is a complete list, so we have to dump and load
|
298
308
|
OfacSdn.delete_all
|
299
309
|
|
300
310
|
puts "Importing into Mysql..."
|
301
|
-
|
311
|
+
yield "Importing into Mysql..." if block_given?
|
312
|
+
|
302
313
|
mysql_command = <<-TEXT
|
303
314
|
LOAD DATA LOCAL INFILE '#{csv_file.path}' REPLACE INTO TABLE ofac_sdns FIELDS TERMINATED BY '|' ENCLOSED BY "`" LINES TERMINATED BY '\n';
|
304
315
|
TEXT
|
305
316
|
|
306
317
|
OfacSdn.connection.execute(mysql_command)
|
307
318
|
puts "Mysql import complete."
|
319
|
+
yield "Mysql import complete." if block_given?
|
308
320
|
|
309
321
|
end
|
310
322
|
|
data/ofac.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{ofac}
|
8
|
-
s.version = "1.2.
|
8
|
+
s.version = "1.2.2"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Kevin Tyll"]
|
12
|
-
s.date = %q{
|
12
|
+
s.date = %q{2011-03-07}
|
13
13
|
s.description = %q{Attempts to find a hit on the Office of Foreign Assets Control's Specially Designated Nationals list.}
|
14
14
|
s.email = %q{kevintyll@gmail.com}
|
15
15
|
s.extra_rdoc_files = [
|
@@ -113,7 +113,7 @@
|
|
113
113
|
|
114
114
|
<div class="method-heading">
|
115
115
|
<a href="#M000009" class="method-signature">
|
116
|
-
<span class="method-name">load_current_sdn_file</span><span class="method-args">()</span>
|
116
|
+
<span class="method-name">load_current_sdn_file</span><span class="method-args">() {|"Downloading OFAC data from http://www.treas.gov/offices/enforcement/ofac/sdn" if block_given?| ...}</span>
|
117
117
|
</a>
|
118
118
|
</div>
|
119
119
|
|
@@ -130,12 +130,14 @@ href="http://www.treas.gov/offices/enforcement/ofac/sdn/delimit/index.shtml">www
|
|
130
130
|
<span class="ruby-keyword kw">def</span> <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">load_current_sdn_file</span>
|
131
131
|
<span class="ruby-identifier">puts</span> <span class="ruby-value str">"Reloading OFAC sdn data"</span>
|
132
132
|
<span class="ruby-identifier">puts</span> <span class="ruby-value str">"Downloading OFAC data from http://www.treas.gov/offices/enforcement/ofac/sdn"</span>
|
133
|
+
<span class="ruby-keyword kw">yield</span> <span class="ruby-value str">"Downloading OFAC data from http://www.treas.gov/offices/enforcement/ofac/sdn"</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">block_given?</span>
|
133
134
|
<span class="ruby-comment cmt">#get the 3 data files</span>
|
134
|
-
<span class="ruby-identifier">sdn</span>
|
135
|
+
<span class="ruby-identifier">sdn</span> = <span class="ruby-constant">Tempfile</span>.<span class="ruby-identifier">new</span>(<span class="ruby-value str">'sdn'</span>)
|
135
136
|
<span class="ruby-identifier">bytes</span> = <span class="ruby-identifier">sdn</span>.<span class="ruby-identifier">write</span>(<span class="ruby-constant">Net</span><span class="ruby-operator">::</span><span class="ruby-constant">HTTP</span>.<span class="ruby-identifier">get</span>(<span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-value str">'http://www.treasury.gov/ofac/downloads/sdn.pip'</span>)))
|
136
137
|
<span class="ruby-identifier">sdn</span>.<span class="ruby-identifier">rewind</span>
|
137
138
|
<span class="ruby-keyword kw">if</span> <span class="ruby-identifier">bytes</span> <span class="ruby-operator">==</span> <span class="ruby-value">0</span> <span class="ruby-operator">||</span> <span class="ruby-identifier">convert_line_to_array</span>(<span class="ruby-identifier">sdn</span>.<span class="ruby-identifier">readline</span>).<span class="ruby-identifier">size</span> <span class="ruby-operator">!=</span> <span class="ruby-value">12</span>
|
138
139
|
<span class="ruby-identifier">puts</span> <span class="ruby-value str">"Trouble downloading file. The url may have changed."</span>
|
140
|
+
<span class="ruby-keyword kw">yield</span> <span class="ruby-value str">"Trouble downloading file. The url may have changed."</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">block_given?</span>
|
139
141
|
<span class="ruby-keyword kw">return</span>
|
140
142
|
<span class="ruby-keyword kw">else</span>
|
141
143
|
<span class="ruby-identifier">sdn</span>.<span class="ruby-identifier">rewind</span>
|
@@ -146,9 +148,10 @@ href="http://www.treas.gov/offices/enforcement/ofac/sdn/delimit/index.shtml">www
|
|
146
148
|
<span class="ruby-identifier">alt</span> = <span class="ruby-constant">Tempfile</span>.<span class="ruby-identifier">new</span>(<span class="ruby-value str">'sdn'</span>)
|
147
149
|
<span class="ruby-identifier">alt</span>.<span class="ruby-identifier">write</span>(<span class="ruby-constant">Net</span><span class="ruby-operator">::</span><span class="ruby-constant">HTTP</span>.<span class="ruby-identifier">get</span>(<span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-value str">'http://www.treasury.gov/ofac/downloads/alt.pip'</span>)))
|
148
150
|
<span class="ruby-identifier">alt</span>.<span class="ruby-identifier">rewind</span>
|
149
|
-
|
151
|
+
|
150
152
|
<span class="ruby-keyword kw">if</span> <span class="ruby-constant">OfacSdn</span>.<span class="ruby-identifier">connection</span>.<span class="ruby-identifier">kind_of?</span>(<span class="ruby-constant">ActiveRecord</span><span class="ruby-operator">::</span><span class="ruby-constant">ConnectionAdapters</span><span class="ruby-operator">::</span><span class="ruby-constant">MysqlAdapter</span>)
|
151
153
|
<span class="ruby-identifier">puts</span> <span class="ruby-value str">"Converting file to csv format for Mysql import. This could take several minutes."</span>
|
154
|
+
<span class="ruby-keyword kw">yield</span> <span class="ruby-value str">"Converting file to csv format for Mysql import. This could take several minutes."</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">block_given?</span>
|
152
155
|
|
153
156
|
<span class="ruby-identifier">csv_file</span> = <span class="ruby-identifier">convert_to_flattened_csv</span>(<span class="ruby-identifier">sdn</span>, <span class="ruby-identifier">address</span>, <span class="ruby-identifier">alt</span>)
|
154
157
|
|
data/rdoc/created.rid
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
Mon, 07 Mar 2011 14:20:44 -0500
|
data/rdoc/index.html
CHANGED
@@ -5,12 +5,12 @@
|
|
5
5
|
|
6
6
|
<!--
|
7
7
|
|
8
|
-
ofac 1.2.
|
8
|
+
ofac 1.2.2
|
9
9
|
|
10
10
|
-->
|
11
11
|
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
12
12
|
<head>
|
13
|
-
<title>ofac 1.2.
|
13
|
+
<title>ofac 1.2.2</title>
|
14
14
|
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
15
15
|
</head>
|
16
16
|
<frameset rows="20%, 80%">
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 1
|
7
7
|
- 2
|
8
|
-
-
|
9
|
-
version: 1.2.
|
8
|
+
- 2
|
9
|
+
version: 1.2.2
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Kevin Tyll
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date:
|
17
|
+
date: 2011-03-07 00:00:00 -05:00
|
18
18
|
default_executable:
|
19
19
|
dependencies: []
|
20
20
|
|