sushi_counter 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/sushi_counter.rb +291 -0
- data/sushi_counter.rb +327 -0
- metadata +90 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: c887dfdb40f6bcc9f4365e6db790c79c187d7d62
|
4
|
+
data.tar.gz: 9887a751bf58b979c0857facff0a2522c54fb296
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: e84b56c117927189ed262586818a051110bc5733c671100a4c9b713a6cd1f519386743f50591dfa050408ca1d64d7bc0498a2f0dca208ea91e659f65b5792c5a
|
7
|
+
data.tar.gz: b5f1fa348245c03d2ba85dfdfb071162d14110acfbd8f54eced1ba2dc733d4fb249f4859964a2a22a365db06cc3b2ed73cbddbf4f14917f76a18938c4c6492f2
|
@@ -0,0 +1,291 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require 'optparse'
|
3
|
+
require 'Savon'
|
4
|
+
require 'csv'
|
5
|
+
require 'nokogiri'
|
6
|
+
require 'date'
|
7
|
+
|
8
|
+
#Creating options, flags and switches for CLI app.
|
9
|
+
options = {}
|
10
|
+
option_parser = OptionParser.new do |opts|
|
11
|
+
executable_name = File.basename($PROGRAM_NAME)
|
12
|
+
opts.banner = "Fetch COUNTER usage data through SUSHI. For SUSHI connection
|
13
|
+
info see here: http://www.niso.org/workrooms/sushi/registry_server/
|
14
|
+
|
15
|
+
Usage: #{executable_name} [options] output_file.csv
|
16
|
+
|
17
|
+
"
|
18
|
+
opts.on("-u [URL]",
|
19
|
+
"URL is the url to the WSDL file specified by the vendor. It will always end in ?wsdl. If you are passing an endpoint, leave this blank."
|
20
|
+
) do |url|
|
21
|
+
options[:url] = url
|
22
|
+
#Check if a WSDL was passed, if not, put in generic WSDL.
|
23
|
+
if options[:url].nil?
|
24
|
+
options[:url] = "http://www.niso.org/schemas/sushi/counter_sushi4_0.wsdl"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
opts.on("-E ENDPOINT",
|
28
|
+
"Endpoint is the url to the service specified by the vendor. It will end with anything but ?wsdl."
|
29
|
+
) do |endpoint|
|
30
|
+
options[:endpoint] = endpoint
|
31
|
+
end
|
32
|
+
opts.on("-r REQUESTOR",
|
33
|
+
"Requestor ID varies from vendor to vendor. See here: http://www.niso.org/workrooms/sushi/registry_server/"
|
34
|
+
) do |requestor|
|
35
|
+
options[:requestor] = requestor
|
36
|
+
end
|
37
|
+
opts.on("-c CUSTOMER",
|
38
|
+
"Customer ID varies from vendor to vendor. See here: http://www.niso.org/workrooms/sushi/registry_server/"
|
39
|
+
) do |customer|
|
40
|
+
options[:customer] = customer
|
41
|
+
end
|
42
|
+
opts.on("-s REPORT_START",
|
43
|
+
"Start date of report in YYYY-MM-DD format.",
|
44
|
+
#RegEx to validate date not working. Need to fix.
|
45
|
+
#/^\d{4}\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$/
|
46
|
+
) do |report_start|
|
47
|
+
options[:report_start] = report_start
|
48
|
+
end
|
49
|
+
opts.on("-e REPORT_END",
|
50
|
+
"End date of report in YYYY-MM-DD format. DD must be the end of the month (e.g. 31)",
|
51
|
+
#RegEx to validate date not working. Need to fix
|
52
|
+
#/^\d{4}\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$/
|
53
|
+
) do |report_end|
|
54
|
+
options[:report_end] = report_end
|
55
|
+
end
|
56
|
+
opts.on("-t TYPE",
|
57
|
+
"Specify which type of COUNTER report you want. If left empty, the default is JR1."
|
58
|
+
) do |type|
|
59
|
+
options[:type] = type
|
60
|
+
if options[:type].nil?
|
61
|
+
options[:type] = "JR1"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
opts.on("-p PASSWORD",
|
65
|
+
"Specify your requestor email/password if required by the vendor."
|
66
|
+
) do |password|
|
67
|
+
options[:password] = password
|
68
|
+
if options[:password].nil?
|
69
|
+
options[:password] = "Blank"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
#Add switch for logging on/off?
|
73
|
+
end
|
74
|
+
begin
|
75
|
+
option_parser.parse!
|
76
|
+
if ARGV.empty?
|
77
|
+
puts "error: you must supply an output_file in format 'output_file.csv'"
|
78
|
+
puts
|
79
|
+
puts option_parser.help
|
80
|
+
exit 1
|
81
|
+
else
|
82
|
+
output_file = ARGV[0]
|
83
|
+
end
|
84
|
+
rescue OptionParser::InvalidArgument => ex
|
85
|
+
STDERR.puts ex.message
|
86
|
+
STDERR.puts option_parser
|
87
|
+
end
|
88
|
+
if ARGV.include? 'DEBUG'
|
89
|
+
puts "url: #{options[:url]}"
|
90
|
+
puts "endpoint: #{options[:endpoint]}"
|
91
|
+
puts "requestor: #{options[:requestor]}"
|
92
|
+
puts "customer: #{options[:customer]}"
|
93
|
+
puts "report_start: #{options[:report_start]}"
|
94
|
+
puts "report_end: #{options[:report_end]}"
|
95
|
+
end
|
96
|
+
|
97
|
+
|
98
|
+
define_method :sushi_call do
|
99
|
+
#Set up client for connection
|
100
|
+
client = Savon.client(
|
101
|
+
#This needs to be passed as a variable. Store all WSDLs as an array and allow
|
102
|
+
#users to select from array?
|
103
|
+
wsdl: "#{options[:url]}",
|
104
|
+
endpoint: "#{options[:endpoint]}",
|
105
|
+
#I think these can stay static, need to validate against a different WSDL. May
|
106
|
+
#need to add additional namespaces?
|
107
|
+
namespaces:{
|
108
|
+
"xmlns:tns" => "SushiService",
|
109
|
+
"xmlns:sc" => "http://www.niso.org/schemas/sushi/counter",
|
110
|
+
"xmlns:xsd" => "http://www.w3.org/2001/XMLSchema",
|
111
|
+
"xmlns:soapenv" => "http://schemas.xmlsoap.org/soap/envelope/",
|
112
|
+
"xmlns:sus" => "http://www.niso.org/schemas/sushi",
|
113
|
+
"xmlns:soap" => "http://schemas.xmlsoap.org/wsdl/soap",
|
114
|
+
"xmlns:soap12" => "http://schemas.xmlsoap.org/wsdl/soap12/"
|
115
|
+
},
|
116
|
+
#Probably keep this static so users don't have to mess with their keys. But need to check
|
117
|
+
#if all keys are the same in each WSDL.
|
118
|
+
convert_request_keys_to: :none,
|
119
|
+
#Logging info. Allow users to set this as a flag to output more/less info. Default should
|
120
|
+
#be minimal.
|
121
|
+
log: true,
|
122
|
+
log_level: :debug,
|
123
|
+
#Allow users to control if they desire - outputs XML to terminal. Default should be FALSE.
|
124
|
+
#Pretty Print is dependent on logging being on
|
125
|
+
pretty_print_xml: true,
|
126
|
+
#Mandatory? I think so. Keep static.
|
127
|
+
env_namespace: :soapenv
|
128
|
+
)
|
129
|
+
#Send out call for data, passing all parameters for login/data
|
130
|
+
#Update to newer Ruby hash style, but how to pass colons in this style?
|
131
|
+
response = client.call(:get_report, message: {
|
132
|
+
"sus:Requestor" => {
|
133
|
+
"sus:ID" => "#{options[:requestor]}",
|
134
|
+
#Two lines below not necessary for connection. BUT. Some vendors will reject
|
135
|
+
#if null. So leave as "blank"
|
136
|
+
"sus:Name" => "Blank",
|
137
|
+
"sus:Email" => "#{options[:password]}"
|
138
|
+
},
|
139
|
+
"sus:CustomerReference" => {
|
140
|
+
"sus:ID" => "#{options[:customer]}"
|
141
|
+
},
|
142
|
+
"sus:ReportDefinition" => {
|
143
|
+
"sus:Filters" => {
|
144
|
+
"sus:UsageDateRange" => {
|
145
|
+
"sus:Begin" => "#{options[:report_start]}",
|
146
|
+
"sus:End" => "#{options[:report_end]}"
|
147
|
+
},
|
148
|
+
},
|
149
|
+
},
|
150
|
+
:attributes! => {
|
151
|
+
"sus:ReportDefinition" => {
|
152
|
+
"Release" => "4",
|
153
|
+
"Name" => "#{options[:type]}"
|
154
|
+
},
|
155
|
+
},
|
156
|
+
} )
|
157
|
+
|
158
|
+
xml = response.doc
|
159
|
+
#Need to replace below xml file with a dynamically generated name based on vendor/year.
|
160
|
+
file = File.open("#{options[:customer]}-#{Time.now.strftime("%Y%m%d")}.txt", "w+")
|
161
|
+
File.write(file, xml.to_xml)
|
162
|
+
|
163
|
+
#Strip out first line of text to allow nokogiri to parse. Not sure if necessary anymore.
|
164
|
+
text = ''
|
165
|
+
File.open(file, "r"){|f|f.gets;text=f.read}
|
166
|
+
File.open(file, "w+"){|f|f.write(text)}
|
167
|
+
|
168
|
+
noko_doc = Nokogiri::XML(File.open(file))
|
169
|
+
#Remove namespaces to access data easier
|
170
|
+
noko_doc.remove_namespaces!
|
171
|
+
|
172
|
+
#Math to get number of months.
|
173
|
+
r_start = options[:report_start]
|
174
|
+
r_end = options[:report_end]
|
175
|
+
define_method(:months_math) do |date1, date2|
|
176
|
+
m = (date2.year * 12 + date2.month) - (date1.year * 12 + date1.month) + 1
|
177
|
+
end
|
178
|
+
#Need to turn report_start and report_end into something readable.
|
179
|
+
date_start = Date.parse(r_start)
|
180
|
+
date_end = Date.parse(r_end)
|
181
|
+
months_var = months_math(date_start, date_end)
|
182
|
+
count_var = months_var - 1
|
183
|
+
|
184
|
+
#Basic info about report.
|
185
|
+
doc_requestor = noko_doc.xpath('//Requestor/ID').text
|
186
|
+
doc_customer_ref = noko_doc.xpath('//CustomerReference/ID').text
|
187
|
+
doc_release = noko_doc.xpath('//ReportDefinition').attr('Release').text
|
188
|
+
doc_version = noko_doc.xpath('//ReportDefinition').attr('Name').text
|
189
|
+
|
190
|
+
|
191
|
+
#Setting empty arrays and variables to manage scope
|
192
|
+
pdf_stats = []
|
193
|
+
pdf_holder = []
|
194
|
+
html_stats = []
|
195
|
+
total_stats = []
|
196
|
+
report_data = []
|
197
|
+
html_holder = []
|
198
|
+
month_holder = []
|
199
|
+
month_count = []
|
200
|
+
month_iterator = nil
|
201
|
+
total_iterator = nil
|
202
|
+
pdf_integer = nil
|
203
|
+
html_integer = nil
|
204
|
+
pdf_iterator = nil
|
205
|
+
html_iterator = nil
|
206
|
+
iterator = nil
|
207
|
+
total_pdf = nil
|
208
|
+
|
209
|
+
noko_doc.xpath('//ReportItems').each do |item|
|
210
|
+
doi = item.xpath("./ItemIdentifier[Type = 'DOI']/Value").text
|
211
|
+
value = item.xpath("./ItemIdentifier[Type = 'Proprietary']/Value").text
|
212
|
+
print_issn = item.xpath("./ItemIdentifier[Type = 'Print_ISSN']/Value").text
|
213
|
+
online_issn = item.xpath("./ItemIdentifier[Type = 'Online_ISSN']/Value").text
|
214
|
+
platform = item.xpath("./ItemPlatform").text
|
215
|
+
publisher = item.xpath("./ItemPublisher").text
|
216
|
+
name = item.xpath("./ItemName").text
|
217
|
+
|
218
|
+
#Probably don't need the line below
|
219
|
+
datatype = item.xpath("./ItemDataType").text
|
220
|
+
|
221
|
+
#Grab usage data in one large array to get sliced below.
|
222
|
+
stats = item.xpath("./ItemPerformance").each do |month|
|
223
|
+
months = month.xpath("./Period/Begin").text
|
224
|
+
#Refactor code to validate against month, and if month missing add a 0 to data
|
225
|
+
count_html = month.xpath("./Instance[MetricType = 'ft_html']/Count").text
|
226
|
+
count_pdf = month.xpath("./Instance[MetricType = 'ft_pdf']/Count").text
|
227
|
+
count = month.xpath("./Instance[MetricType = 'ft_total']/Count").text
|
228
|
+
#If no data for that month write a 0 instead of blank
|
229
|
+
if count.empty?
|
230
|
+
count = "0"
|
231
|
+
else count_html = count_html
|
232
|
+
end
|
233
|
+
month_holder << months
|
234
|
+
|
235
|
+
#Printing out of months/dates not working correctly. Need to format headers
|
236
|
+
#based off of data passed, not data from XML (XML often poorly formatted)
|
237
|
+
month_count = month_holder[0..count_var]
|
238
|
+
month_iterator = month_count.join(",")
|
239
|
+
pdf_stats << count_pdf
|
240
|
+
html_stats << count_html
|
241
|
+
total_stats << count
|
242
|
+
total_pdf = pdf_stats.map(&:to_i).reduce(:+)
|
243
|
+
end
|
244
|
+
|
245
|
+
#Refine code below? Currently working, but could probably be DRYer
|
246
|
+
total_stats.each_slice(months_var).each do |slice|
|
247
|
+
iterator = slice.join(",")
|
248
|
+
end
|
249
|
+
#Could probably just add up html_stats array and output in CSV row below.
|
250
|
+
html_stats.each_slice(months_var).each do |slice|
|
251
|
+
html_holder = slice
|
252
|
+
html_integer = html_holder.map(&:to_i)
|
253
|
+
html_iterator = html_integer.inject(0, :+)
|
254
|
+
end
|
255
|
+
pdf_stats.each_slice(months_var).each do |slice|
|
256
|
+
pdf_holder = slice
|
257
|
+
pdf_integer = pdf_holder.map(&:to_i)
|
258
|
+
pdf_iterator = pdf_integer.inject(0, :+)
|
259
|
+
end
|
260
|
+
total_iterator = pdf_iterator + html_iterator
|
261
|
+
#All data needs to fit into this array.
|
262
|
+
#Iterator data doesn't break out to new columns despite having comma, as it's inserted as one block of data.
|
263
|
+
report_data << [name, publisher, platform, doi, value, print_issn, online_issn, total_iterator, html_iterator, pdf_iterator, iterator]
|
264
|
+
end
|
265
|
+
|
266
|
+
|
267
|
+
##Output Nokogiri Doc into csv
|
268
|
+
CSV.open(output_file, 'wb') do |row|
|
269
|
+
row << ["#{doc_version}", "Release: #{doc_release}"]
|
270
|
+
row << ["Requestor ID: #{doc_requestor}", " Customer ID: #{doc_customer_ref}"]
|
271
|
+
row << ["Period covered by Report:"]
|
272
|
+
row << ["#{options[:report_start]} to #{options[:report_end]}"]
|
273
|
+
row << ["Date run:"]
|
274
|
+
row << ["#{Time.now.strftime("%d/%m/%Y")}"]
|
275
|
+
|
276
|
+
#This row needs to include month/year for each month. So need to get number
|
277
|
+
# of months from above (slice parentheses value) and year from input. Also, commas aren't automatically spacing out to new column when opened in CSV.
|
278
|
+
row << ["Journal", "Publisher", "Platform", "Journal DOI", "Proprietary Identifier", "Print ISSN", "Online ISSN", "Reporting Period Total", "Reporting Period HTML", "Reporting Period PDF", month_iterator]
|
279
|
+
row << [total_pdf]
|
280
|
+
#Add line that totals all data?
|
281
|
+
# row <<
|
282
|
+
|
283
|
+
#Code below iterates through array, printing each item to a row
|
284
|
+
report_data.each do |data|
|
285
|
+
row << data
|
286
|
+
end
|
287
|
+
end
|
288
|
+
#Need to rearrange columns as data isn't coming out sequentially.
|
289
|
+
#CSV.open(output_file, 'wb', :headers => true)
|
290
|
+
end
|
291
|
+
sushi_call()
|
data/sushi_counter.rb
ADDED
@@ -0,0 +1,327 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require 'optparse'
|
3
|
+
require 'Savon'
|
4
|
+
require 'csv'
|
5
|
+
require 'nokogiri'
|
6
|
+
require 'date'
|
7
|
+
|
8
|
+
#Creating options, flags and switches for CLI app.
|
9
|
+
options = {}
|
10
|
+
option_parser = OptionParser.new do |opts|
|
11
|
+
executable_name = File.basename($PROGRAM_NAME)
|
12
|
+
opts.banner = "Fetch COUNTER usage data through SUSHI. For SUSHI connection
|
13
|
+
info see here: http://www.niso.org/workrooms/sushi/registry_server/
|
14
|
+
|
15
|
+
Usage: #{executable_name} [options] output_file.csv
|
16
|
+
|
17
|
+
"
|
18
|
+
opts.on("-u [URL]",
|
19
|
+
"URL is the url to the WSDL file specified by the vendor. It will always end in ?wsdl. If you are passing an endpoint, leave this blank."
|
20
|
+
) do |url|
|
21
|
+
options[:url] = url
|
22
|
+
#Check if a WSDL was passed, if not, put in generic WSDL.
|
23
|
+
if options[:url].nil?
|
24
|
+
options[:url] = "http://www.niso.org/schemas/sushi/counter_sushi4_0.wsdl"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
opts.on("-E ENDPOINT",
|
28
|
+
"Endpoint is the url to the service specified by the vendor. It will end with anything but ?wsdl."
|
29
|
+
) do |endpoint|
|
30
|
+
options[:endpoint] = endpoint
|
31
|
+
end
|
32
|
+
opts.on("-r REQUESTOR",
|
33
|
+
"Requestor ID varies from vendor to vendor. See here: http://www.niso.org/workrooms/sushi/registry_server/"
|
34
|
+
) do |requestor|
|
35
|
+
options[:requestor] = requestor
|
36
|
+
end
|
37
|
+
opts.on("-c CUSTOMER",
|
38
|
+
"Customer ID varies from vendor to vendor. See here: http://www.niso.org/workrooms/sushi/registry_server/"
|
39
|
+
) do |customer|
|
40
|
+
options[:customer] = customer
|
41
|
+
end
|
42
|
+
opts.on("-s REPORT_START",
|
43
|
+
"Start date of report in YYYY-MM-DD format.",
|
44
|
+
#RegEx to validate date not working. Need to fix.
|
45
|
+
#/^\d{4}\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$/
|
46
|
+
) do |report_start|
|
47
|
+
options[:report_start] = report_start
|
48
|
+
end
|
49
|
+
opts.on("-e REPORT_END",
|
50
|
+
"End date of report in YYYY-MM-DD format. DD must be the end of the month (e.g. 31)",
|
51
|
+
#RegEx to validate date not working. Need to fix
|
52
|
+
#/^\d{4}\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$/
|
53
|
+
) do |report_end|
|
54
|
+
options[:report_end] = report_end
|
55
|
+
end
|
56
|
+
opts.on("-t TYPE",
|
57
|
+
"Specify which type of COUNTER report you want. If left empty, the default is JR1."
|
58
|
+
) do |type|
|
59
|
+
options[:type] = type
|
60
|
+
if options[:type].nil?
|
61
|
+
options[:type] = "JR1"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
opts.on("-p [PASSWORD]",
|
65
|
+
"Specify your requestor email/password if required by the vendor."
|
66
|
+
) do |password|
|
67
|
+
options[:password] = password
|
68
|
+
if options[:password].nil?
|
69
|
+
options[:password] = "Blank"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
#Add switch for logging on/off?
|
73
|
+
end
|
74
|
+
begin
|
75
|
+
option_parser.parse!
|
76
|
+
if ARGV.empty?
|
77
|
+
puts "error: you must supply an output_file in format 'output_file.csv'"
|
78
|
+
puts
|
79
|
+
puts option_parser.help
|
80
|
+
exit 1
|
81
|
+
else
|
82
|
+
output_file = ARGV[0]
|
83
|
+
end
|
84
|
+
rescue OptionParser::InvalidArgument => ex
|
85
|
+
STDERR.puts ex.message
|
86
|
+
STDERR.puts option_parser
|
87
|
+
end
|
88
|
+
if ARGV.include? 'DEBUG'
|
89
|
+
puts "The parameters you are passing to each option are:"
|
90
|
+
puts "url: #{options[:url]}"
|
91
|
+
puts "endpoint: #{options[:endpoint]}"
|
92
|
+
puts "requestor: #{options[:requestor]}"
|
93
|
+
puts "customer: #{options[:customer]}"
|
94
|
+
puts "report_start: #{options[:report_start]}"
|
95
|
+
puts "report_end: #{options[:report_end]}"
|
96
|
+
puts "password: #{options[:password]}"
|
97
|
+
end
|
98
|
+
|
99
|
+
|
100
|
+
define_method :sushi_call do
|
101
|
+
#Set up client for connection
|
102
|
+
client = Savon.client(
|
103
|
+
#This needs to be passed as a variable. Store all WSDLs as an array and allow
|
104
|
+
#users to select from array?
|
105
|
+
wsdl: "#{options[:url]}",
|
106
|
+
endpoint: "#{options[:endpoint]}",
|
107
|
+
#I think these can stay static, need to validate against a different WSDL. May
|
108
|
+
#need to add additional namespaces?
|
109
|
+
namespaces:{
|
110
|
+
"xmlns:tns" => "SushiService",
|
111
|
+
"xmlns:sc" => "http://www.niso.org/schemas/sushi/counter",
|
112
|
+
"xmlns:xsd" => "http://www.w3.org/2001/XMLSchema",
|
113
|
+
"xmlns:soapenv" => "http://schemas.xmlsoap.org/soap/envelope/",
|
114
|
+
"xmlns:sus" => "http://www.niso.org/schemas/sushi",
|
115
|
+
"xmlns:soap" => "http://schemas.xmlsoap.org/wsdl/soap",
|
116
|
+
"xmlns:soap12" => "http://schemas.xmlsoap.org/wsdl/soap12/"
|
117
|
+
},
|
118
|
+
#Probably keep this static so users don't have to mess with their keys. But need to check
|
119
|
+
#if all keys are the same in each WSDL.
|
120
|
+
convert_request_keys_to: :none,
|
121
|
+
#Logging info. Allow users to set this as a flag to output more/less info. Default should
|
122
|
+
#be minimal.
|
123
|
+
log: true,
|
124
|
+
log_level: :debug,
|
125
|
+
#Allow users to control if they desire - outputs XML to terminal. Default should be FALSE.
|
126
|
+
#Pretty Print is dependent on logging being on
|
127
|
+
pretty_print_xml: true,
|
128
|
+
#Mandatory? I think so. Keep static.
|
129
|
+
env_namespace: :soapenv
|
130
|
+
)
|
131
|
+
#Send out call for data, passing all parameters for login/data
|
132
|
+
#Update to newer Ruby hash style, but how to pass colons in this style?
|
133
|
+
response = client.call(:get_report, message: {
|
134
|
+
"sus:Requestor" => {
|
135
|
+
"sus:ID" => "#{options[:requestor]}",
|
136
|
+
#Two lines below not necessary for connection. BUT. Some vendors will reject
|
137
|
+
#if null. So leave as "blank"
|
138
|
+
"sus:Name" => "Blank",
|
139
|
+
"sus:Email" => "#{options[:password]}"
|
140
|
+
},
|
141
|
+
"sus:CustomerReference" => {
|
142
|
+
"sus:ID" => "#{options[:customer]}"
|
143
|
+
},
|
144
|
+
"sus:ReportDefinition" => {
|
145
|
+
"sus:Filters" => {
|
146
|
+
"sus:UsageDateRange" => {
|
147
|
+
"sus:Begin" => "#{options[:report_start]}",
|
148
|
+
"sus:End" => "#{options[:report_end]}"
|
149
|
+
},
|
150
|
+
},
|
151
|
+
},
|
152
|
+
:attributes! => {
|
153
|
+
"sus:ReportDefinition" => {
|
154
|
+
"Release" => "4",
|
155
|
+
"Name" => "#{options[:type]}"
|
156
|
+
},
|
157
|
+
},
|
158
|
+
} )
|
159
|
+
|
160
|
+
xml = response.doc
|
161
|
+
#Need to replace below xml file with a dynamically generated name based on vendor/year.
|
162
|
+
file = File.open("raw_xml/#{options[:customer]}-#{Time.now.strftime("%Y%m%d")}.xml", "w+")
|
163
|
+
File.write(file, xml.to_xml)
|
164
|
+
|
165
|
+
#Strip out first line of text to allow nokogiri to parse. Not sure if necessary anymore.
|
166
|
+
text = ''
|
167
|
+
File.open(file, "r"){|f|f.gets;text=f.read}
|
168
|
+
File.open(file, "w+"){|f|f.write(text)}
|
169
|
+
|
170
|
+
noko_doc = Nokogiri::XML(File.open(file))
|
171
|
+
#Remove namespaces to access data easier
|
172
|
+
noko_doc.remove_namespaces!
|
173
|
+
|
174
|
+
month_array = []
|
175
|
+
#Math to get number of months and print out to headers
|
176
|
+
r_start = options[:report_start]
|
177
|
+
r_end = options[:report_end]
|
178
|
+
define_method(:months_math) do |date1, date2|
|
179
|
+
m = (date2.year * 12 + date2.month) - (date1.year * 12 + date1.month) + 1
|
180
|
+
m.times do |n|
|
181
|
+
iterator = date1 >> n
|
182
|
+
month_array << iterator.to_s
|
183
|
+
end
|
184
|
+
end
|
185
|
+
#Turn report_start and report_end into something readable.
|
186
|
+
date_start = Date.parse(r_start)
|
187
|
+
date_end = Date.parse(r_end)
|
188
|
+
months_var = months_math(date_start, date_end)
|
189
|
+
count_var = months_var - 1
|
190
|
+
|
191
|
+
#Basic info about report.
|
192
|
+
doc_requestor = noko_doc.xpath('//Requestor/ID').text
|
193
|
+
doc_customer_ref = noko_doc.xpath('//CustomerReference/ID').text
|
194
|
+
doc_release = noko_doc.xpath('//ReportDefinition').attr('Release').text
|
195
|
+
doc_version = noko_doc.xpath('//ReportDefinition').attr('Name').text
|
196
|
+
|
197
|
+
|
198
|
+
#Setting empty arrays and variables to manage scope
|
199
|
+
pdf_stats = []
|
200
|
+
pdf_holder = []
|
201
|
+
html_stats = []
|
202
|
+
total_stats = []
|
203
|
+
report_data = []
|
204
|
+
html_holder = []
|
205
|
+
month_holder = []
|
206
|
+
count_holder = []
|
207
|
+
count_hash = Hash[month_array.map{|x| [x.to_sym] }]
|
208
|
+
month_iterator = nil
|
209
|
+
total_iterator = nil
|
210
|
+
pdf_integer = nil
|
211
|
+
html_integer = nil
|
212
|
+
pdf_iterator = nil
|
213
|
+
html_iterator = nil
|
214
|
+
iterator = nil
|
215
|
+
total_pdf = nil
|
216
|
+
i = 0
|
217
|
+
|
218
|
+
|
219
|
+
noko_doc.xpath('//ReportItems').each do |item|
|
220
|
+
doi = item.xpath("./ItemIdentifier[Type = 'DOI']/Value").text
|
221
|
+
value = item.xpath("./ItemIdentifier[Type = 'Proprietary']/Value").text
|
222
|
+
print_issn = item.xpath("./ItemIdentifier[Type = 'Print_ISSN']/Value").text
|
223
|
+
online_issn = item.xpath("./ItemIdentifier[Type = 'Online_ISSN']/Value").text
|
224
|
+
platform = item.xpath("./ItemPlatform").text
|
225
|
+
publisher = item.xpath("./ItemPublisher").text
|
226
|
+
name = item.xpath("./ItemName").text
|
227
|
+
|
228
|
+
#Probably don't need the line below
|
229
|
+
datatype = item.xpath("./ItemDataType").text
|
230
|
+
|
231
|
+
#Store all months from XML data into an array
|
232
|
+
month_match = item.xpath("./ItemPerformance/Period").each do |match|
|
233
|
+
matches = match.xpath("./Begin").text
|
234
|
+
month_holder << matches
|
235
|
+
end
|
236
|
+
count_values = item.xpath("./ItemPerformance").each do |count|
|
237
|
+
month = count.xpath("./Period/Begin").text
|
238
|
+
counts = count.xpath("./Instance[MetricType = 'ft_total']/Count").text
|
239
|
+
if counts.empty?
|
240
|
+
counts = "0"
|
241
|
+
else counts = counts
|
242
|
+
end
|
243
|
+
count_hash[month.to_sym] = counts
|
244
|
+
end
|
245
|
+
|
246
|
+
n = 0
|
247
|
+
m = 0
|
248
|
+
|
249
|
+
|
250
|
+
#Match months supplied by user to months in month_holder array
|
251
|
+
#and grab usage stats for those months
|
252
|
+
months_var.times do
|
253
|
+
if month_holder.include? month_array[n]
|
254
|
+
month_key = month_array[n]
|
255
|
+
count = count_hash[month_key.to_sym]
|
256
|
+
#i += 1
|
257
|
+
m += 1
|
258
|
+
total_stats << count
|
259
|
+
if n < count_var
|
260
|
+
n += 1
|
261
|
+
else n = 0
|
262
|
+
end
|
263
|
+
else
|
264
|
+
fallback_count = "test"
|
265
|
+
total_stats << fallback_count
|
266
|
+
if n < count_var
|
267
|
+
n += 1
|
268
|
+
else n = 0
|
269
|
+
end
|
270
|
+
end
|
271
|
+
stats = item.xpath("./ItemPerformance").each do |month|
|
272
|
+
count_html = month.xpath("./Instance[MetricType = 'ft_html']/Count").text
|
273
|
+
count_pdf = month.xpath("./Instance[MetricType = 'ft_pdf']/Count").text
|
274
|
+
pdf_stats << count_pdf
|
275
|
+
html_stats << count_html
|
276
|
+
|
277
|
+
#Add total columns here. Need more (html, all, each month, etc...)
|
278
|
+
total_pdf = pdf_stats.map(&:to_i).reduce(:+)
|
279
|
+
end
|
280
|
+
end
|
281
|
+
|
282
|
+
#Make code below DRYer
|
283
|
+
#Refine code below? Currently working, but could probably be DRYer
|
284
|
+
total_stats.each_slice(months_var).each do |slice|
|
285
|
+
iterator = slice.join(",")
|
286
|
+
end
|
287
|
+
#Could probably just add up html_stats array and output in CSV row below.
|
288
|
+
html_stats.each_slice(months_var).each do |slice|
|
289
|
+
html_holder = slice
|
290
|
+
html_integer = html_holder.map(&:to_i)
|
291
|
+
html_iterator = html_integer.inject(0, :+)
|
292
|
+
end
|
293
|
+
pdf_stats.each_slice(months_var).each do |slice|
|
294
|
+
pdf_holder = slice
|
295
|
+
pdf_integer = pdf_holder.map(&:to_i)
|
296
|
+
pdf_iterator = pdf_integer.inject(0, :+)
|
297
|
+
end
|
298
|
+
total_iterator = pdf_iterator + html_iterator
|
299
|
+
|
300
|
+
#All data needs to fit into this array.
|
301
|
+
#Iterator data doesn't break out to new columns despite having comma, as it's inserted as one block of data.
|
302
|
+
report_data << [name, publisher, platform, doi, value, print_issn, online_issn, total_iterator, html_iterator, pdf_iterator, iterator]
|
303
|
+
end
|
304
|
+
|
305
|
+
##Output Nokogiri Doc into csv
|
306
|
+
CSV.open(output_file, 'wb') do |row|
|
307
|
+
row << ["#{doc_version}", "Release: #{doc_release}"]
|
308
|
+
row << ["Requestor ID: #{doc_requestor}", " Customer ID: #{doc_customer_ref}"]
|
309
|
+
row << ["Period covered by Report:"]
|
310
|
+
row << ["#{options[:report_start]} to #{options[:report_end]}"]
|
311
|
+
row << ["Date run:"]
|
312
|
+
row << ["#{Time.now.strftime("%d/%m/%Y")}"]
|
313
|
+
|
314
|
+
#This row needs to include month/year for each month. So need to get number
|
315
|
+
# of months from above (slice parentheses value) and year from input. Also, commas aren't automatically spacing out to new column when opened in CSV.
|
316
|
+
row << ["Journal", "Publisher", "Platform", "Journal DOI", "Proprietary Identifier", "Print ISSN", "Online ISSN", "Reporting Period Total", "Reporting Period HTML", "Reporting Period PDF", month_array]
|
317
|
+
row << [total_pdf]
|
318
|
+
#Add line that totals all data?
|
319
|
+
# row <<
|
320
|
+
|
321
|
+
#Code below iterates through array, printing each item to a row
|
322
|
+
report_data.each do |data|
|
323
|
+
row << data
|
324
|
+
end
|
325
|
+
end
|
326
|
+
end
|
327
|
+
sushi_call()
|
metadata
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sushi_counter
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Max King
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-10-19 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: savon
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: nokogiri
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: aruba
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.5.3
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.5.3
|
55
|
+
description: |-
|
56
|
+
Sushi fetches the raw XML from a number of vendors, then
|
57
|
+
converts that data into a human readable CSV
|
58
|
+
email:
|
59
|
+
- maxdavidking at gmail.com
|
60
|
+
executables:
|
61
|
+
- sushi_counter.rb
|
62
|
+
extensions: []
|
63
|
+
extra_rdoc_files: []
|
64
|
+
files:
|
65
|
+
- bin/sushi_counter.rb
|
66
|
+
- sushi_counter.rb
|
67
|
+
homepage: http://maxdavidking.com
|
68
|
+
licenses: []
|
69
|
+
metadata: {}
|
70
|
+
post_install_message:
|
71
|
+
rdoc_options: []
|
72
|
+
require_paths:
|
73
|
+
- lib
|
74
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
75
|
+
requirements:
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: '0'
|
79
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
84
|
+
requirements: []
|
85
|
+
rubyforge_project: sushi
|
86
|
+
rubygems_version: 2.6.13
|
87
|
+
signing_key:
|
88
|
+
specification_version: 4
|
89
|
+
summary: A SUSHI/COUNTER data retrieval app
|
90
|
+
test_files: []
|