datacatalog-importer 0.1.8 → 0.1.9
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/datacatalog-importer.gemspec +2 -2
- data/lib/pusher.rb +26 -51
- metadata +3 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.9
|
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{datacatalog-importer}
|
8
|
-
s.version = "0.1.
|
8
|
+
s.version = "0.1.9"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["David James"]
|
12
|
-
s.date = %q{2010-03-
|
12
|
+
s.date = %q{2010-03-17}
|
13
13
|
s.description = %q{This framework makes it easier to write importers for the National Data Catalog.}
|
14
14
|
s.email = %q{djames@sunlightfoundation.com}
|
15
15
|
s.extra_rdoc_files = [
|
data/lib/pusher.rb
CHANGED
@@ -91,23 +91,22 @@ module DataCatalog
|
|
91
91
|
puts "Updating Organization: #{name}"
|
92
92
|
DataCatalog::Organization.update(docs[0].id, data)
|
93
93
|
else
|
94
|
-
|
94
|
+
error("Cannot find unique Source with url : #{url}", {
|
95
|
+
:error => "#{n} matches: " + docs.map { |x| x.id }.join(" ")
|
96
|
+
})
|
95
97
|
end
|
96
98
|
end
|
97
99
|
|
98
100
|
def find_organization_by(field, name)
|
99
|
-
|
100
|
-
DataCatalog::Organization.all(field => name)
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
def verify_one_result(field, name, model)
|
105
|
-
docs = yield
|
101
|
+
docs = DataCatalog::Organization.all(field => name)
|
106
102
|
n = docs.length
|
107
103
|
case n
|
108
104
|
when 0 then nil
|
109
105
|
when 1 then docs[0]
|
110
|
-
else
|
106
|
+
else
|
107
|
+
error("Cannot find unique Organization with #{field} : #{name}", {
|
108
|
+
:error => "#{n} matches: " + docs.map { |x| x.id }.join(" ")
|
109
|
+
})
|
111
110
|
end
|
112
111
|
end
|
113
112
|
|
@@ -122,7 +121,9 @@ module DataCatalog
|
|
122
121
|
begin
|
123
122
|
DataCatalog::Source.create(data)
|
124
123
|
rescue DataCatalog::BadRequest => e
|
125
|
-
|
124
|
+
error("Cannot create Source", {
|
125
|
+
:params => data
|
126
|
+
})
|
126
127
|
end
|
127
128
|
when 1
|
128
129
|
puts "Updating Source: #{data[:title]}"
|
@@ -130,10 +131,14 @@ module DataCatalog
|
|
130
131
|
id = docs[0].id
|
131
132
|
DataCatalog::Source.update(id, data)
|
132
133
|
rescue DataCatalog::BadRequest => e
|
133
|
-
|
134
|
+
error("Cannot update Source with id : #{id}", {
|
135
|
+
:params => data
|
136
|
+
})
|
134
137
|
end
|
135
138
|
else
|
136
|
-
|
139
|
+
error("Cannot find unique Source with url : #{data[:url]}", {
|
140
|
+
:error => "#{n} matches: " + docs.map { |x| x.id }.join(" ")
|
141
|
+
})
|
137
142
|
end
|
138
143
|
end
|
139
144
|
|
@@ -153,7 +158,9 @@ module DataCatalog
|
|
153
158
|
puts "- Updating Download: #{data[:format]}"
|
154
159
|
DataCatalog::Download.update(docs[0].id, data)
|
155
160
|
else
|
156
|
-
|
161
|
+
error("Cannot find unique Download with source_id : #{source.id} and format : #{data[:format]}", {
|
162
|
+
:error => "#{n} matches: " + docs.map { |x| x.id }.join(" ")
|
163
|
+
})
|
157
164
|
end
|
158
165
|
end
|
159
166
|
|
@@ -175,7 +182,7 @@ module DataCatalog
|
|
175
182
|
keys.each do |key|
|
176
183
|
value = hash[key]
|
177
184
|
organization = if value
|
178
|
-
attempts << "#{key}
|
185
|
+
attempts << "#{key} : #{value}"
|
179
186
|
find_organization_by(key, value)
|
180
187
|
end
|
181
188
|
break if organization
|
@@ -189,47 +196,15 @@ module DataCatalog
|
|
189
196
|
true # return value not important
|
190
197
|
end
|
191
198
|
|
192
|
-
|
193
|
-
|
194
|
-
def create_failed(model, text, data, error)
|
195
|
-
puts "- Failed. #{error}"
|
196
|
-
puts " Uploading Report to API."
|
197
|
-
DataCatalog::Report.create({
|
198
|
-
:status => "new",
|
199
|
-
:text => "Cannot create #{model} : #{text}",
|
200
|
-
:object => {
|
201
|
-
:error => error,
|
202
|
-
:params => data,
|
203
|
-
},
|
204
|
-
})
|
205
|
-
nil
|
206
|
-
end
|
207
|
-
|
208
|
-
def update_failed(model, data, id, error)
|
209
|
-
puts "- Failed. #{error}"
|
210
|
-
puts " Uploading Report to API."
|
211
|
-
DataCatalog::Report.create({
|
212
|
-
:status => "new",
|
213
|
-
:text => "Cannot update #{model} of id : #{id}",
|
214
|
-
:object => {
|
215
|
-
:error => error,
|
216
|
-
:params => data,
|
217
|
-
},
|
218
|
-
})
|
219
|
-
nil
|
220
|
-
end
|
199
|
+
protected
|
221
200
|
|
222
|
-
def
|
223
|
-
puts "-
|
201
|
+
def error(text, object)
|
202
|
+
puts "- Failure: #{text}"
|
224
203
|
puts " Uploading Report to API."
|
225
204
|
DataCatalog::Report.create({
|
226
205
|
:status => "new",
|
227
|
-
:text =>
|
228
|
-
:object =>
|
229
|
-
:error => "Cannot automatically update #{model} since there " +
|
230
|
-
"are #{n} matches for url : #{data[:url]}",
|
231
|
-
:params => data,
|
232
|
-
},
|
206
|
+
:text => text,
|
207
|
+
:object => object,
|
233
208
|
})
|
234
209
|
nil
|
235
210
|
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 1
|
8
|
-
-
|
9
|
-
version: 0.1.
|
8
|
+
- 9
|
9
|
+
version: 0.1.9
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- David James
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2010-03-
|
17
|
+
date: 2010-03-17 00:00:00 -04:00
|
18
18
|
default_executable:
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|