datacatalog-importer 0.1.6 → 0.1.7
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/datacatalog-importer.gemspec +2 -2
- data/lib/pusher.rb +67 -20
- metadata +3 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.7
|
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{datacatalog-importer}
|
8
|
-
s.version = "0.1.
|
8
|
+
s.version = "0.1.7"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["David James"]
|
12
|
-
s.date = %q{2010-03-
|
12
|
+
s.date = %q{2010-03-12}
|
13
13
|
s.description = %q{This framework makes it easier to write importers for the National Data Catalog.}
|
14
14
|
s.email = %q{djames@sunlightfoundation.com}
|
15
15
|
s.extra_rdoc_files = [
|
data/lib/pusher.rb
CHANGED
@@ -43,8 +43,13 @@ module DataCatalog
|
|
43
43
|
read_data(:source) do |data|
|
44
44
|
link_to_existing_organization!(data, :organization_id)
|
45
45
|
source = create_or_update_source(data)
|
46
|
-
|
47
|
-
|
46
|
+
if source
|
47
|
+
downloads = data[:downloads]
|
48
|
+
if downloads
|
49
|
+
downloads.each do |download_data|
|
50
|
+
create_or_update_download(source, download_data)
|
51
|
+
end
|
52
|
+
end
|
48
53
|
end
|
49
54
|
end
|
50
55
|
end
|
@@ -82,7 +87,7 @@ module DataCatalog
|
|
82
87
|
puts "Updating Organization: #{name}"
|
83
88
|
DataCatalog::Organization.update(docs[0].id, data)
|
84
89
|
else
|
85
|
-
multiple_matches("Organization",
|
90
|
+
multiple_matches("Organization", data, n)
|
86
91
|
end
|
87
92
|
end
|
88
93
|
|
@@ -95,8 +100,7 @@ module DataCatalog
|
|
95
100
|
when 1
|
96
101
|
docs[0]
|
97
102
|
else
|
98
|
-
multiple_matches("Organization",
|
99
|
-
nil
|
103
|
+
multiple_matches("Organization", { field => name }, n)
|
100
104
|
end
|
101
105
|
end
|
102
106
|
|
@@ -105,18 +109,26 @@ module DataCatalog
|
|
105
109
|
data = data.reject do |key, value|
|
106
110
|
[:organization, :downloads].include?(key)
|
107
111
|
end
|
108
|
-
puts "data[:url] : #{data[:url]}"
|
109
112
|
docs = DataCatalog::Source.all(:url => data[:url])
|
110
113
|
n = docs.length
|
111
114
|
case n
|
112
115
|
when 0
|
113
116
|
puts "Creating Source: #{data[:title]}"
|
114
|
-
|
117
|
+
begin
|
118
|
+
DataCatalog::Source.create(data)
|
119
|
+
rescue DataCatalog::BadRequest => e
|
120
|
+
create_failed("Source", data[:title], data, e)
|
121
|
+
end
|
115
122
|
when 1
|
116
123
|
puts "Updating Source: #{data[:title]}"
|
117
|
-
|
124
|
+
begin
|
125
|
+
id = docs[0].id
|
126
|
+
DataCatalog::Source.update(id, data)
|
127
|
+
rescue DataCatalog::BadRequest => e
|
128
|
+
update_failed("Source", data, id, e)
|
129
|
+
end
|
118
130
|
else
|
119
|
-
multiple_matches("Source",
|
131
|
+
multiple_matches("Source", data, n)
|
120
132
|
end
|
121
133
|
end
|
122
134
|
|
@@ -136,7 +148,7 @@ module DataCatalog
|
|
136
148
|
puts "- Updating Download: #{data[:format]}"
|
137
149
|
DataCatalog::Download.update(docs[0].id, data)
|
138
150
|
else
|
139
|
-
multiple_matches("Download",
|
151
|
+
multiple_matches("Download", data, n)
|
140
152
|
end
|
141
153
|
end
|
142
154
|
|
@@ -149,29 +161,64 @@ module DataCatalog
|
|
149
161
|
raise "Could not find :organization key" unless organization_hash
|
150
162
|
name = organization_hash[:name]
|
151
163
|
url = organization_hash[:url]
|
152
|
-
|
153
|
-
|
164
|
+
|
165
|
+
organization, error_message = if url
|
166
|
+
[find_organization_by(:url, url), "url : #{url}"]
|
154
167
|
elsif name
|
155
|
-
find_organization_by(:name, name)
|
168
|
+
[find_organization_by(:name, name), "name : #{name}"]
|
156
169
|
end
|
157
|
-
|
170
|
+
|
158
171
|
if organization
|
159
172
|
data[organization_id_key] = organization.id
|
160
173
|
else
|
161
|
-
puts "
|
174
|
+
puts " - Could not find organization with #{error_message}"
|
162
175
|
end
|
176
|
+
true # return value not important
|
163
177
|
end
|
164
178
|
|
165
179
|
# ---
|
166
180
|
|
167
|
-
def
|
168
|
-
puts "
|
181
|
+
def create_failed(model, text, data, error)
|
182
|
+
puts " - Failed. #{error}"
|
183
|
+
puts " Uploading Report to API."
|
184
|
+
DataCatalog::Report.create({
|
185
|
+
:status => "new",
|
186
|
+
:text => "Cannot create #{model} : #{text}",
|
187
|
+
:object => {
|
188
|
+
:error => error,
|
189
|
+
:params => data,
|
190
|
+
},
|
191
|
+
})
|
192
|
+
nil
|
193
|
+
end
|
194
|
+
|
195
|
+
def update_failed(model, data, id, error)
|
196
|
+
puts " - Failed. #{error}"
|
197
|
+
puts " Uploading Report to API."
|
198
|
+
DataCatalog::Report.create({
|
199
|
+
:status => "new",
|
200
|
+
:text => "Cannot update #{model} of id : #{id}",
|
201
|
+
:object => {
|
202
|
+
:error => error,
|
203
|
+
:params => data,
|
204
|
+
},
|
205
|
+
})
|
206
|
+
nil
|
207
|
+
end
|
208
|
+
|
209
|
+
def multiple_matches(model, data, n)
|
210
|
+
puts " - Failed. #{n} matches for #{model}."
|
211
|
+
puts " Uploading Report to API."
|
169
212
|
DataCatalog::Report.create({
|
170
213
|
:status => "new",
|
171
|
-
:text => "
|
172
|
-
|
173
|
-
|
214
|
+
:text => "Multiple matches for url : #{data[:url]}",
|
215
|
+
:object => {
|
216
|
+
:error => "Cannot automatically update #{model}since there " +
|
217
|
+
"are #{n} matches for url : #{data[:url]}",
|
218
|
+
:params => data,
|
219
|
+
},
|
174
220
|
})
|
221
|
+
nil
|
175
222
|
end
|
176
223
|
|
177
224
|
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 1
|
8
|
-
-
|
9
|
-
version: 0.1.
|
8
|
+
- 7
|
9
|
+
version: 0.1.7
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- David James
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2010-03-
|
17
|
+
date: 2010-03-12 00:00:00 -05:00
|
18
18
|
default_executable:
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|