sushi_counter 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/sushi_counter.rb +291 -0
- data/sushi_counter.rb +327 -0
- metadata +90 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: c887dfdb40f6bcc9f4365e6db790c79c187d7d62
|
4
|
+
data.tar.gz: 9887a751bf58b979c0857facff0a2522c54fb296
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: e84b56c117927189ed262586818a051110bc5733c671100a4c9b713a6cd1f519386743f50591dfa050408ca1d64d7bc0498a2f0dca208ea91e659f65b5792c5a
|
7
|
+
data.tar.gz: b5f1fa348245c03d2ba85dfdfb071162d14110acfbd8f54eced1ba2dc733d4fb249f4859964a2a22a365db06cc3b2ed73cbddbf4f14917f76a18938c4c6492f2
|
@@ -0,0 +1,291 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require 'optparse'
|
3
|
+
require 'Savon'
|
4
|
+
require 'csv'
|
5
|
+
require 'nokogiri'
|
6
|
+
require 'date'
|
7
|
+
|
8
|
+
#Creating options, flags and switches for CLI app.
|
9
|
+
options = {}
|
10
|
+
option_parser = OptionParser.new do |opts|
|
11
|
+
executable_name = File.basename($PROGRAM_NAME)
|
12
|
+
opts.banner = "Fetch COUNTER usage data through SUSHI. For SUSHI connection
|
13
|
+
info see here: http://www.niso.org/workrooms/sushi/registry_server/
|
14
|
+
|
15
|
+
Usage: #{executable_name} [options] output_file.csv
|
16
|
+
|
17
|
+
"
|
18
|
+
opts.on("-u [URL]",
|
19
|
+
"URL is the url to the WSDL file specified by the vendor. It will always end in ?wsdl. If you are passing an endpoint, leave this blank."
|
20
|
+
) do |url|
|
21
|
+
options[:url] = url
|
22
|
+
#Check if a WSDL was passed, if not, put in generic WSDL.
|
23
|
+
if options[:url].nil?
|
24
|
+
options[:url] = "http://www.niso.org/schemas/sushi/counter_sushi4_0.wsdl"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
opts.on("-E ENDPOINT",
|
28
|
+
"Endpoint is the url to the service specified by the vendor. It will end with anything but ?wsdl."
|
29
|
+
) do |endpoint|
|
30
|
+
options[:endpoint] = endpoint
|
31
|
+
end
|
32
|
+
opts.on("-r REQUESTOR",
|
33
|
+
"Requestor ID varies from vendor to vendor. See here: http://www.niso.org/workrooms/sushi/registry_server/"
|
34
|
+
) do |requestor|
|
35
|
+
options[:requestor] = requestor
|
36
|
+
end
|
37
|
+
opts.on("-c CUSTOMER",
|
38
|
+
"Customer ID varies from vendor to vendor. See here: http://www.niso.org/workrooms/sushi/registry_server/"
|
39
|
+
) do |customer|
|
40
|
+
options[:customer] = customer
|
41
|
+
end
|
42
|
+
opts.on("-s REPORT_START",
|
43
|
+
"Start date of report in YYYY-MM-DD format.",
|
44
|
+
#RegEx to validate date not working. Need to fix.
|
45
|
+
#/^\d{4}\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$/
|
46
|
+
) do |report_start|
|
47
|
+
options[:report_start] = report_start
|
48
|
+
end
|
49
|
+
opts.on("-e REPORT_END",
|
50
|
+
"End date of report in YYYY-MM-DD format. DD must be the end of the month (e.g. 31)",
|
51
|
+
#RegEx to validate date not working. Need to fix
|
52
|
+
#/^\d{4}\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$/
|
53
|
+
) do |report_end|
|
54
|
+
options[:report_end] = report_end
|
55
|
+
end
|
56
|
+
opts.on("-t TYPE",
|
57
|
+
"Specify which type of COUNTER report you want. If left empty, the default is JR1."
|
58
|
+
) do |type|
|
59
|
+
options[:type] = type
|
60
|
+
if options[:type].nil?
|
61
|
+
options[:type] = "JR1"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
opts.on("-p PASSWORD",
|
65
|
+
"Specify your requestor email/password if required by the vendor."
|
66
|
+
) do |password|
|
67
|
+
options[:password] = password
|
68
|
+
if options[:password].nil?
|
69
|
+
options[:password] = "Blank"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
#Add switch for logging on/off?
|
73
|
+
end
|
74
|
+
begin
|
75
|
+
option_parser.parse!
|
76
|
+
if ARGV.empty?
|
77
|
+
puts "error: you must supply an output_file in format 'output_file.csv'"
|
78
|
+
puts
|
79
|
+
puts option_parser.help
|
80
|
+
exit 1
|
81
|
+
else
|
82
|
+
output_file = ARGV[0]
|
83
|
+
end
|
84
|
+
rescue OptionParser::InvalidArgument => ex
|
85
|
+
STDERR.puts ex.message
|
86
|
+
STDERR.puts option_parser
|
87
|
+
end
|
88
|
+
if ARGV.include? 'DEBUG'
|
89
|
+
puts "url: #{options[:url]}"
|
90
|
+
puts "endpoint: #{options[:endpoint]}"
|
91
|
+
puts "requestor: #{options[:requestor]}"
|
92
|
+
puts "customer: #{options[:customer]}"
|
93
|
+
puts "report_start: #{options[:report_start]}"
|
94
|
+
puts "report_end: #{options[:report_end]}"
|
95
|
+
end
|
96
|
+
|
97
|
+
|
98
|
+
define_method :sushi_call do
|
99
|
+
#Set up client for connection
|
100
|
+
client = Savon.client(
|
101
|
+
#This needs to be passed as a variable. Store all WSDLs as an array and allow
|
102
|
+
#users to select from array?
|
103
|
+
wsdl: "#{options[:url]}",
|
104
|
+
endpoint: "#{options[:endpoint]}",
|
105
|
+
#I think these can stay static, need to validate against a different WSDL. May
|
106
|
+
#need to add additional namespaces?
|
107
|
+
namespaces:{
|
108
|
+
"xmlns:tns" => "SushiService",
|
109
|
+
"xmlns:sc" => "http://www.niso.org/schemas/sushi/counter",
|
110
|
+
"xmlns:xsd" => "http://www.w3.org/2001/XMLSchema",
|
111
|
+
"xmlns:soapenv" => "http://schemas.xmlsoap.org/soap/envelope/",
|
112
|
+
"xmlns:sus" => "http://www.niso.org/schemas/sushi",
|
113
|
+
"xmlns:soap" => "http://schemas.xmlsoap.org/wsdl/soap",
|
114
|
+
"xmlns:soap12" => "http://schemas.xmlsoap.org/wsdl/soap12/"
|
115
|
+
},
|
116
|
+
#Probably keep this static so users don't have to mess with their keys. But need to check
|
117
|
+
#if all keys are the same in each WSDL.
|
118
|
+
convert_request_keys_to: :none,
|
119
|
+
#Logging info. Allow users to set this as a flag to output more/less info. Default should
|
120
|
+
#be minimal.
|
121
|
+
log: true,
|
122
|
+
log_level: :debug,
|
123
|
+
#Allow users to control if they desire - outputs XML to terminal. Default should be FALSE.
|
124
|
+
#Pretty Print is dependent on logging being on
|
125
|
+
pretty_print_xml: true,
|
126
|
+
#Mandatory? I think so. Keep static.
|
127
|
+
env_namespace: :soapenv
|
128
|
+
)
|
129
|
+
#Send out call for data, passing all parameters for login/data
|
130
|
+
#Update to newer Ruby hash style, but how to pass colons in this style?
|
131
|
+
response = client.call(:get_report, message: {
|
132
|
+
"sus:Requestor" => {
|
133
|
+
"sus:ID" => "#{options[:requestor]}",
|
134
|
+
#Two lines below not necessary for connection. BUT. Some vendors will reject
|
135
|
+
#if null. So leave as "blank"
|
136
|
+
"sus:Name" => "Blank",
|
137
|
+
"sus:Email" => "#{options[:password]}"
|
138
|
+
},
|
139
|
+
"sus:CustomerReference" => {
|
140
|
+
"sus:ID" => "#{options[:customer]}"
|
141
|
+
},
|
142
|
+
"sus:ReportDefinition" => {
|
143
|
+
"sus:Filters" => {
|
144
|
+
"sus:UsageDateRange" => {
|
145
|
+
"sus:Begin" => "#{options[:report_start]}",
|
146
|
+
"sus:End" => "#{options[:report_end]}"
|
147
|
+
},
|
148
|
+
},
|
149
|
+
},
|
150
|
+
:attributes! => {
|
151
|
+
"sus:ReportDefinition" => {
|
152
|
+
"Release" => "4",
|
153
|
+
"Name" => "#{options[:type]}"
|
154
|
+
},
|
155
|
+
},
|
156
|
+
} )
|
157
|
+
|
158
|
+
xml = response.doc
|
159
|
+
#Need to replace below xml file with a dynamically generated name based on vendor/year.
|
160
|
+
file = File.open("#{options[:customer]}-#{Time.now.strftime("%Y%m%d")}.txt", "w+")
|
161
|
+
File.write(file, xml.to_xml)
|
162
|
+
|
163
|
+
#Strip out first line of text to allow nokogiri to parse. Not sure if necessary anymore.
|
164
|
+
text = ''
|
165
|
+
File.open(file, "r"){|f|f.gets;text=f.read}
|
166
|
+
File.open(file, "w+"){|f|f.write(text)}
|
167
|
+
|
168
|
+
noko_doc = Nokogiri::XML(File.open(file))
|
169
|
+
#Remove namespaces to access data easier
|
170
|
+
noko_doc.remove_namespaces!
|
171
|
+
|
172
|
+
#Math to get number of months.
|
173
|
+
r_start = options[:report_start]
|
174
|
+
r_end = options[:report_end]
|
175
|
+
define_method(:months_math) do |date1, date2|
|
176
|
+
m = (date2.year * 12 + date2.month) - (date1.year * 12 + date1.month) + 1
|
177
|
+
end
|
178
|
+
#Need to turn report_start and report_end into something readable.
|
179
|
+
date_start = Date.parse(r_start)
|
180
|
+
date_end = Date.parse(r_end)
|
181
|
+
months_var = months_math(date_start, date_end)
|
182
|
+
count_var = months_var - 1
|
183
|
+
|
184
|
+
#Basic info about report.
|
185
|
+
doc_requestor = noko_doc.xpath('//Requestor/ID').text
|
186
|
+
doc_customer_ref = noko_doc.xpath('//CustomerReference/ID').text
|
187
|
+
doc_release = noko_doc.xpath('//ReportDefinition').attr('Release').text
|
188
|
+
doc_version = noko_doc.xpath('//ReportDefinition').attr('Name').text
|
189
|
+
|
190
|
+
|
191
|
+
#Setting empty arrays and variables to manage scope
|
192
|
+
pdf_stats = []
|
193
|
+
pdf_holder = []
|
194
|
+
html_stats = []
|
195
|
+
total_stats = []
|
196
|
+
report_data = []
|
197
|
+
html_holder = []
|
198
|
+
month_holder = []
|
199
|
+
month_count = []
|
200
|
+
month_iterator = nil
|
201
|
+
total_iterator = nil
|
202
|
+
pdf_integer = nil
|
203
|
+
html_integer = nil
|
204
|
+
pdf_iterator = nil
|
205
|
+
html_iterator = nil
|
206
|
+
iterator = nil
|
207
|
+
total_pdf = nil
|
208
|
+
|
209
|
+
noko_doc.xpath('//ReportItems').each do |item|
|
210
|
+
doi = item.xpath("./ItemIdentifier[Type = 'DOI']/Value").text
|
211
|
+
value = item.xpath("./ItemIdentifier[Type = 'Proprietary']/Value").text
|
212
|
+
print_issn = item.xpath("./ItemIdentifier[Type = 'Print_ISSN']/Value").text
|
213
|
+
online_issn = item.xpath("./ItemIdentifier[Type = 'Online_ISSN']/Value").text
|
214
|
+
platform = item.xpath("./ItemPlatform").text
|
215
|
+
publisher = item.xpath("./ItemPublisher").text
|
216
|
+
name = item.xpath("./ItemName").text
|
217
|
+
|
218
|
+
#Probably don't need the line below
|
219
|
+
datatype = item.xpath("./ItemDataType").text
|
220
|
+
|
221
|
+
#Grab usage data in one large array to get sliced below.
|
222
|
+
stats = item.xpath("./ItemPerformance").each do |month|
|
223
|
+
months = month.xpath("./Period/Begin").text
|
224
|
+
#Refactor code to validate against month, and if month missing add a 0 to data
|
225
|
+
count_html = month.xpath("./Instance[MetricType = 'ft_html']/Count").text
|
226
|
+
count_pdf = month.xpath("./Instance[MetricType = 'ft_pdf']/Count").text
|
227
|
+
count = month.xpath("./Instance[MetricType = 'ft_total']/Count").text
|
228
|
+
#If no data for that month write a 0 instead of blank
|
229
|
+
if count.empty?
|
230
|
+
count = "0"
|
231
|
+
else count_html = count_html
|
232
|
+
end
|
233
|
+
month_holder << months
|
234
|
+
|
235
|
+
#Printing out of months/dates not working correctly. Need to format headers
|
236
|
+
#based off of data passed, not data from XML (XML often poorly formatted)
|
237
|
+
month_count = month_holder[0..count_var]
|
238
|
+
month_iterator = month_count.join(",")
|
239
|
+
pdf_stats << count_pdf
|
240
|
+
html_stats << count_html
|
241
|
+
total_stats << count
|
242
|
+
total_pdf = pdf_stats.map(&:to_i).reduce(:+)
|
243
|
+
end
|
244
|
+
|
245
|
+
#Refine code below? Currently working, but could probably be DRYer
|
246
|
+
total_stats.each_slice(months_var).each do |slice|
|
247
|
+
iterator = slice.join(",")
|
248
|
+
end
|
249
|
+
#Could probably just add up html_stats array and output in CSV row below.
|
250
|
+
html_stats.each_slice(months_var).each do |slice|
|
251
|
+
html_holder = slice
|
252
|
+
html_integer = html_holder.map(&:to_i)
|
253
|
+
html_iterator = html_integer.inject(0, :+)
|
254
|
+
end
|
255
|
+
pdf_stats.each_slice(months_var).each do |slice|
|
256
|
+
pdf_holder = slice
|
257
|
+
pdf_integer = pdf_holder.map(&:to_i)
|
258
|
+
pdf_iterator = pdf_integer.inject(0, :+)
|
259
|
+
end
|
260
|
+
total_iterator = pdf_iterator + html_iterator
|
261
|
+
#All data needs to fit into this array.
|
262
|
+
#Iterator data doesn't break out to new columns despite having comma, as it's inserted as one block of data.
|
263
|
+
report_data << [name, publisher, platform, doi, value, print_issn, online_issn, total_iterator, html_iterator, pdf_iterator, iterator]
|
264
|
+
end
|
265
|
+
|
266
|
+
|
267
|
+
##Output Nokogiri Doc into csv
|
268
|
+
CSV.open(output_file, 'wb') do |row|
|
269
|
+
row << ["#{doc_version}", "Release: #{doc_release}"]
|
270
|
+
row << ["Requestor ID: #{doc_requestor}", " Customer ID: #{doc_customer_ref}"]
|
271
|
+
row << ["Period covered by Report:"]
|
272
|
+
row << ["#{options[:report_start]} to #{options[:report_end]}"]
|
273
|
+
row << ["Date run:"]
|
274
|
+
row << ["#{Time.now.strftime("%d/%m/%Y")}"]
|
275
|
+
|
276
|
+
#This row needs to include month/year for each month. So need to get number
|
277
|
+
# of months from above (slice parentheses value) and year from input. Also, commas aren't automatically spacing out to new column when opened in CSV.
|
278
|
+
row << ["Journal", "Publisher", "Platform", "Journal DOI", "Proprietary Identifier", "Print ISSN", "Online ISSN", "Reporting Period Total", "Reporting Period HTML", "Reporting Period PDF", month_iterator]
|
279
|
+
row << [total_pdf]
|
280
|
+
#Add line that totals all data?
|
281
|
+
# row <<
|
282
|
+
|
283
|
+
#Code below iterates through array, printing each item to a row
|
284
|
+
report_data.each do |data|
|
285
|
+
row << data
|
286
|
+
end
|
287
|
+
end
|
288
|
+
#Need to rearrange columns as data isn't coming out sequentially.
|
289
|
+
#CSV.open(output_file, 'wb', :headers => true)
|
290
|
+
end
|
291
|
+
sushi_call()
|
data/sushi_counter.rb
ADDED
@@ -0,0 +1,327 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require 'optparse'
|
3
|
+
require 'Savon'
|
4
|
+
require 'csv'
|
5
|
+
require 'nokogiri'
|
6
|
+
require 'date'
|
7
|
+
|
8
|
+
#Creating options, flags and switches for CLI app.
|
9
|
+
options = {}
|
10
|
+
option_parser = OptionParser.new do |opts|
|
11
|
+
executable_name = File.basename($PROGRAM_NAME)
|
12
|
+
opts.banner = "Fetch COUNTER usage data through SUSHI. For SUSHI connection
|
13
|
+
info see here: http://www.niso.org/workrooms/sushi/registry_server/
|
14
|
+
|
15
|
+
Usage: #{executable_name} [options] output_file.csv
|
16
|
+
|
17
|
+
"
|
18
|
+
opts.on("-u [URL]",
|
19
|
+
"URL is the url to the WSDL file specified by the vendor. It will always end in ?wsdl. If you are passing an endpoint, leave this blank."
|
20
|
+
) do |url|
|
21
|
+
options[:url] = url
|
22
|
+
#Check if a WSDL was passed, if not, put in generic WSDL.
|
23
|
+
if options[:url].nil?
|
24
|
+
options[:url] = "http://www.niso.org/schemas/sushi/counter_sushi4_0.wsdl"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
opts.on("-E ENDPOINT",
|
28
|
+
"Endpoint is the url to the service specified by the vendor. It will end with anything but ?wsdl."
|
29
|
+
) do |endpoint|
|
30
|
+
options[:endpoint] = endpoint
|
31
|
+
end
|
32
|
+
opts.on("-r REQUESTOR",
|
33
|
+
"Requestor ID varies from vendor to vendor. See here: http://www.niso.org/workrooms/sushi/registry_server/"
|
34
|
+
) do |requestor|
|
35
|
+
options[:requestor] = requestor
|
36
|
+
end
|
37
|
+
opts.on("-c CUSTOMER",
|
38
|
+
"Customer ID varies from vendor to vendor. See here: http://www.niso.org/workrooms/sushi/registry_server/"
|
39
|
+
) do |customer|
|
40
|
+
options[:customer] = customer
|
41
|
+
end
|
42
|
+
opts.on("-s REPORT_START",
|
43
|
+
"Start date of report in YYYY-MM-DD format.",
|
44
|
+
#RegEx to validate date not working. Need to fix.
|
45
|
+
#/^\d{4}\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$/
|
46
|
+
) do |report_start|
|
47
|
+
options[:report_start] = report_start
|
48
|
+
end
|
49
|
+
opts.on("-e REPORT_END",
|
50
|
+
"End date of report in YYYY-MM-DD format. DD must be the end of the month (e.g. 31)",
|
51
|
+
#RegEx to validate date not working. Need to fix
|
52
|
+
#/^\d{4}\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$/
|
53
|
+
) do |report_end|
|
54
|
+
options[:report_end] = report_end
|
55
|
+
end
|
56
|
+
opts.on("-t TYPE",
|
57
|
+
"Specify which type of COUNTER report you want. If left empty, the default is JR1."
|
58
|
+
) do |type|
|
59
|
+
options[:type] = type
|
60
|
+
if options[:type].nil?
|
61
|
+
options[:type] = "JR1"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
opts.on("-p [PASSWORD]",
|
65
|
+
"Specify your requestor email/password if required by the vendor."
|
66
|
+
) do |password|
|
67
|
+
options[:password] = password
|
68
|
+
if options[:password].nil?
|
69
|
+
options[:password] = "Blank"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
#Add switch for logging on/off?
|
73
|
+
end
|
74
|
+
begin
|
75
|
+
option_parser.parse!
|
76
|
+
if ARGV.empty?
|
77
|
+
puts "error: you must supply an output_file in format 'output_file.csv'"
|
78
|
+
puts
|
79
|
+
puts option_parser.help
|
80
|
+
exit 1
|
81
|
+
else
|
82
|
+
output_file = ARGV[0]
|
83
|
+
end
|
84
|
+
rescue OptionParser::InvalidArgument => ex
|
85
|
+
STDERR.puts ex.message
|
86
|
+
STDERR.puts option_parser
|
87
|
+
end
|
88
|
+
if ARGV.include? 'DEBUG'
|
89
|
+
puts "The parameters you are passing to each option are:"
|
90
|
+
puts "url: #{options[:url]}"
|
91
|
+
puts "endpoint: #{options[:endpoint]}"
|
92
|
+
puts "requestor: #{options[:requestor]}"
|
93
|
+
puts "customer: #{options[:customer]}"
|
94
|
+
puts "report_start: #{options[:report_start]}"
|
95
|
+
puts "report_end: #{options[:report_end]}"
|
96
|
+
puts "password: #{options[:password]}"
|
97
|
+
end
|
98
|
+
|
99
|
+
|
100
|
+
define_method :sushi_call do
|
101
|
+
#Set up client for connection
|
102
|
+
client = Savon.client(
|
103
|
+
#This needs to be passed as a variable. Store all WSDLs as an array and allow
|
104
|
+
#users to select from array?
|
105
|
+
wsdl: "#{options[:url]}",
|
106
|
+
endpoint: "#{options[:endpoint]}",
|
107
|
+
#I think these can stay static, need to validate against a different WSDL. May
|
108
|
+
#need to add additional namespaces?
|
109
|
+
namespaces:{
|
110
|
+
"xmlns:tns" => "SushiService",
|
111
|
+
"xmlns:sc" => "http://www.niso.org/schemas/sushi/counter",
|
112
|
+
"xmlns:xsd" => "http://www.w3.org/2001/XMLSchema",
|
113
|
+
"xmlns:soapenv" => "http://schemas.xmlsoap.org/soap/envelope/",
|
114
|
+
"xmlns:sus" => "http://www.niso.org/schemas/sushi",
|
115
|
+
"xmlns:soap" => "http://schemas.xmlsoap.org/wsdl/soap",
|
116
|
+
"xmlns:soap12" => "http://schemas.xmlsoap.org/wsdl/soap12/"
|
117
|
+
},
|
118
|
+
#Probably keep this static so users don't have to mess with their keys. But need to check
|
119
|
+
#if all keys are the same in each WSDL.
|
120
|
+
convert_request_keys_to: :none,
|
121
|
+
#Logging info. Allow users to set this as a flag to output more/less info. Default should
|
122
|
+
#be minimal.
|
123
|
+
log: true,
|
124
|
+
log_level: :debug,
|
125
|
+
#Allow users to control if they desire - outputs XML to terminal. Default should be FALSE.
|
126
|
+
#Pretty Print is dependent on logging being on
|
127
|
+
pretty_print_xml: true,
|
128
|
+
#Mandatory? I think so. Keep static.
|
129
|
+
env_namespace: :soapenv
|
130
|
+
)
|
131
|
+
#Send out call for data, passing all parameters for login/data
|
132
|
+
#Update to newer Ruby hash style, but how to pass colons in this style?
|
133
|
+
response = client.call(:get_report, message: {
|
134
|
+
"sus:Requestor" => {
|
135
|
+
"sus:ID" => "#{options[:requestor]}",
|
136
|
+
#Two lines below not necessary for connection. BUT. Some vendors will reject
|
137
|
+
#if null. So leave as "blank"
|
138
|
+
"sus:Name" => "Blank",
|
139
|
+
"sus:Email" => "#{options[:password]}"
|
140
|
+
},
|
141
|
+
"sus:CustomerReference" => {
|
142
|
+
"sus:ID" => "#{options[:customer]}"
|
143
|
+
},
|
144
|
+
"sus:ReportDefinition" => {
|
145
|
+
"sus:Filters" => {
|
146
|
+
"sus:UsageDateRange" => {
|
147
|
+
"sus:Begin" => "#{options[:report_start]}",
|
148
|
+
"sus:End" => "#{options[:report_end]}"
|
149
|
+
},
|
150
|
+
},
|
151
|
+
},
|
152
|
+
:attributes! => {
|
153
|
+
"sus:ReportDefinition" => {
|
154
|
+
"Release" => "4",
|
155
|
+
"Name" => "#{options[:type]}"
|
156
|
+
},
|
157
|
+
},
|
158
|
+
} )
|
159
|
+
|
160
|
+
xml = response.doc
|
161
|
+
#Need to replace below xml file with a dynamically generated name based on vendor/year.
|
162
|
+
file = File.open("raw_xml/#{options[:customer]}-#{Time.now.strftime("%Y%m%d")}.xml", "w+")
|
163
|
+
File.write(file, xml.to_xml)
|
164
|
+
|
165
|
+
#Strip out first line of text to allow nokogiri to parse. Not sure if necessary anymore.
|
166
|
+
text = ''
|
167
|
+
File.open(file, "r"){|f|f.gets;text=f.read}
|
168
|
+
File.open(file, "w+"){|f|f.write(text)}
|
169
|
+
|
170
|
+
noko_doc = Nokogiri::XML(File.open(file))
|
171
|
+
#Remove namespaces to access data easier
|
172
|
+
noko_doc.remove_namespaces!
|
173
|
+
|
174
|
+
month_array = []
|
175
|
+
#Math to get number of months and print out to headers
|
176
|
+
r_start = options[:report_start]
|
177
|
+
r_end = options[:report_end]
|
178
|
+
define_method(:months_math) do |date1, date2|
|
179
|
+
m = (date2.year * 12 + date2.month) - (date1.year * 12 + date1.month) + 1
|
180
|
+
m.times do |n|
|
181
|
+
iterator = date1 >> n
|
182
|
+
month_array << iterator.to_s
|
183
|
+
end
|
184
|
+
end
|
185
|
+
#Turn report_start and report_end into something readable.
|
186
|
+
date_start = Date.parse(r_start)
|
187
|
+
date_end = Date.parse(r_end)
|
188
|
+
months_var = months_math(date_start, date_end)
|
189
|
+
count_var = months_var - 1
|
190
|
+
|
191
|
+
#Basic info about report.
|
192
|
+
doc_requestor = noko_doc.xpath('//Requestor/ID').text
|
193
|
+
doc_customer_ref = noko_doc.xpath('//CustomerReference/ID').text
|
194
|
+
doc_release = noko_doc.xpath('//ReportDefinition').attr('Release').text
|
195
|
+
doc_version = noko_doc.xpath('//ReportDefinition').attr('Name').text
|
196
|
+
|
197
|
+
|
198
|
+
#Setting empty arrays and variables to manage scope
|
199
|
+
pdf_stats = []
|
200
|
+
pdf_holder = []
|
201
|
+
html_stats = []
|
202
|
+
total_stats = []
|
203
|
+
report_data = []
|
204
|
+
html_holder = []
|
205
|
+
month_holder = []
|
206
|
+
count_holder = []
|
207
|
+
count_hash = Hash[month_array.map{|x| [x.to_sym] }]
|
208
|
+
month_iterator = nil
|
209
|
+
total_iterator = nil
|
210
|
+
pdf_integer = nil
|
211
|
+
html_integer = nil
|
212
|
+
pdf_iterator = nil
|
213
|
+
html_iterator = nil
|
214
|
+
iterator = nil
|
215
|
+
total_pdf = nil
|
216
|
+
i = 0
|
217
|
+
|
218
|
+
|
219
|
+
noko_doc.xpath('//ReportItems').each do |item|
|
220
|
+
doi = item.xpath("./ItemIdentifier[Type = 'DOI']/Value").text
|
221
|
+
value = item.xpath("./ItemIdentifier[Type = 'Proprietary']/Value").text
|
222
|
+
print_issn = item.xpath("./ItemIdentifier[Type = 'Print_ISSN']/Value").text
|
223
|
+
online_issn = item.xpath("./ItemIdentifier[Type = 'Online_ISSN']/Value").text
|
224
|
+
platform = item.xpath("./ItemPlatform").text
|
225
|
+
publisher = item.xpath("./ItemPublisher").text
|
226
|
+
name = item.xpath("./ItemName").text
|
227
|
+
|
228
|
+
#Probably don't need the line below
|
229
|
+
datatype = item.xpath("./ItemDataType").text
|
230
|
+
|
231
|
+
#Store all months from XML data into an array
|
232
|
+
month_match = item.xpath("./ItemPerformance/Period").each do |match|
|
233
|
+
matches = match.xpath("./Begin").text
|
234
|
+
month_holder << matches
|
235
|
+
end
|
236
|
+
count_values = item.xpath("./ItemPerformance").each do |count|
|
237
|
+
month = count.xpath("./Period/Begin").text
|
238
|
+
counts = count.xpath("./Instance[MetricType = 'ft_total']/Count").text
|
239
|
+
if counts.empty?
|
240
|
+
counts = "0"
|
241
|
+
else counts = counts
|
242
|
+
end
|
243
|
+
count_hash[month.to_sym] = counts
|
244
|
+
end
|
245
|
+
|
246
|
+
n = 0
|
247
|
+
m = 0
|
248
|
+
|
249
|
+
|
250
|
+
#Match months supplied by user to months in month_holder array
|
251
|
+
#and grab usage stats for those months
|
252
|
+
months_var.times do
|
253
|
+
if month_holder.include? month_array[n]
|
254
|
+
month_key = month_array[n]
|
255
|
+
count = count_hash[month_key.to_sym]
|
256
|
+
#i += 1
|
257
|
+
m += 1
|
258
|
+
total_stats << count
|
259
|
+
if n < count_var
|
260
|
+
n += 1
|
261
|
+
else n = 0
|
262
|
+
end
|
263
|
+
else
|
264
|
+
fallback_count = "test"
|
265
|
+
total_stats << fallback_count
|
266
|
+
if n < count_var
|
267
|
+
n += 1
|
268
|
+
else n = 0
|
269
|
+
end
|
270
|
+
end
|
271
|
+
stats = item.xpath("./ItemPerformance").each do |month|
|
272
|
+
count_html = month.xpath("./Instance[MetricType = 'ft_html']/Count").text
|
273
|
+
count_pdf = month.xpath("./Instance[MetricType = 'ft_pdf']/Count").text
|
274
|
+
pdf_stats << count_pdf
|
275
|
+
html_stats << count_html
|
276
|
+
|
277
|
+
#Add total columns here. Need more (html, all, each month, etc...)
|
278
|
+
total_pdf = pdf_stats.map(&:to_i).reduce(:+)
|
279
|
+
end
|
280
|
+
end
|
281
|
+
|
282
|
+
#Make code below DRYer
|
283
|
+
#Refine code below? Currently working, but could probably be DRYer
|
284
|
+
total_stats.each_slice(months_var).each do |slice|
|
285
|
+
iterator = slice.join(",")
|
286
|
+
end
|
287
|
+
#Could probably just add up html_stats array and output in CSV row below.
|
288
|
+
html_stats.each_slice(months_var).each do |slice|
|
289
|
+
html_holder = slice
|
290
|
+
html_integer = html_holder.map(&:to_i)
|
291
|
+
html_iterator = html_integer.inject(0, :+)
|
292
|
+
end
|
293
|
+
pdf_stats.each_slice(months_var).each do |slice|
|
294
|
+
pdf_holder = slice
|
295
|
+
pdf_integer = pdf_holder.map(&:to_i)
|
296
|
+
pdf_iterator = pdf_integer.inject(0, :+)
|
297
|
+
end
|
298
|
+
total_iterator = pdf_iterator + html_iterator
|
299
|
+
|
300
|
+
#All data needs to fit into this array.
|
301
|
+
#Iterator data doesn't break out to new columns despite having comma, as it's inserted as one block of data.
|
302
|
+
report_data << [name, publisher, platform, doi, value, print_issn, online_issn, total_iterator, html_iterator, pdf_iterator, iterator]
|
303
|
+
end
|
304
|
+
|
305
|
+
##Output Nokogiri Doc into csv
|
306
|
+
CSV.open(output_file, 'wb') do |row|
|
307
|
+
row << ["#{doc_version}", "Release: #{doc_release}"]
|
308
|
+
row << ["Requestor ID: #{doc_requestor}", " Customer ID: #{doc_customer_ref}"]
|
309
|
+
row << ["Period covered by Report:"]
|
310
|
+
row << ["#{options[:report_start]} to #{options[:report_end]}"]
|
311
|
+
row << ["Date run:"]
|
312
|
+
row << ["#{Time.now.strftime("%d/%m/%Y")}"]
|
313
|
+
|
314
|
+
#This row needs to include month/year for each month. So need to get number
|
315
|
+
# of months from above (slice parentheses value) and year from input. Also, commas aren't automatically spacing out to new column when opened in CSV.
|
316
|
+
row << ["Journal", "Publisher", "Platform", "Journal DOI", "Proprietary Identifier", "Print ISSN", "Online ISSN", "Reporting Period Total", "Reporting Period HTML", "Reporting Period PDF", month_array]
|
317
|
+
row << [total_pdf]
|
318
|
+
#Add line that totals all data?
|
319
|
+
# row <<
|
320
|
+
|
321
|
+
#Code below iterates through array, printing each item to a row
|
322
|
+
report_data.each do |data|
|
323
|
+
row << data
|
324
|
+
end
|
325
|
+
end
|
326
|
+
end
|
327
|
+
sushi_call()
|
metadata
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sushi_counter
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Max King
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-10-19 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: savon
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: nokogiri
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: aruba
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.5.3
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.5.3
|
55
|
+
description: |-
|
56
|
+
Sushi fetches the raw XML from a number of vendors, then
|
57
|
+
converts that data into a human readable CSV
|
58
|
+
email:
|
59
|
+
- maxdavidking at gmail.com
|
60
|
+
executables:
|
61
|
+
- sushi_counter.rb
|
62
|
+
extensions: []
|
63
|
+
extra_rdoc_files: []
|
64
|
+
files:
|
65
|
+
- bin/sushi_counter.rb
|
66
|
+
- sushi_counter.rb
|
67
|
+
homepage: http://maxdavidking.com
|
68
|
+
licenses: []
|
69
|
+
metadata: {}
|
70
|
+
post_install_message:
|
71
|
+
rdoc_options: []
|
72
|
+
require_paths:
|
73
|
+
- lib
|
74
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
75
|
+
requirements:
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: '0'
|
79
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
84
|
+
requirements: []
|
85
|
+
rubyforge_project: sushi
|
86
|
+
rubygems_version: 2.6.13
|
87
|
+
signing_key:
|
88
|
+
specification_version: 4
|
89
|
+
summary: A SUSHI/COUNTER data retrieval app
|
90
|
+
test_files: []
|