prpm-tools 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rspec +2 -0
- data/.travis.yml +4 -0
- data/Gemfile +6 -0
- data/README.md +112 -0
- data/Rakefile +6 -0
- data/bin/prpm-parse-logfile +63 -0
- data/bin/prpm-update-botdata +21 -0
- data/lib/bot_filter/bot_data.json +1 -0
- data/lib/bot_filter/bot_filter.rb +19 -0
- data/lib/line_filter.rb +45 -0
- data/lib/log_line_parsers/akamai.rb +43 -0
- data/lib/log_line_parsers/nginx.rb +35 -0
- data/lib/log_line_parsers/triton.rb +44 -0
- data/lib/prpmtools.rb +11 -0
- data/lib/prpmtools/version.rb +3 -0
- data/lib/show_parsers/rnz.rb +17 -0
- data/license.txt +22 -0
- data/prpmtools.gemspec +26 -0
- metadata +118 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 10c68d283dccae45ec829431c068df46f14e398e
|
4
|
+
data.tar.gz: a5464ee5b9c104a705b63eef9aa529226095b3a3
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 95cb1f782be4989180f5c382b04efc2a34b5b56ce9a61477aa4e9451ef92ae5fb01eadbcadfd7a4bde0bdcf0d15e7d985b24db7822d4ebe7b5c801c4e24a62d6
|
7
|
+
data.tar.gz: 621302673c6708a55c07a7913b01bd84c10764b5fccde665bc054c991076089a53054578ceb49651eb3b3f84ea61349711414b73a31422a938cf2b0817b6b574
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,112 @@
|
|
1
|
+
# Public Radio Podcast Measurement Tools
|
2
|
+
|
3
|
+
NB: This tool is still under heavy development - feedback is invited. The gem will be released in due course.
|
4
|
+
|
5
|
+
This gem contains tools for parsing logs according to the PRPM Guidelines:
|
6
|
+
|
7
|
+
https://docs.google.com/document/d/1qMORNRNyAR3L_03VYlvZK9dpKMk_d7pUxCnJfeKgVn0/edit
|
8
|
+
|
9
|
+
The tools can be used from the command line via:
|
10
|
+
|
11
|
+
prpm-parse-logile
|
12
|
+
|
13
|
+
or the underlying classes can be used as part of a Ruby script or Rails app as needed.
|
14
|
+
|
15
|
+
These tools aim to provide a unified output for all input log formats, so that logs can be consolidated, and the parsed files can be imported into other systems for further analysis without any surprises
|
16
|
+
|
17
|
+
The criteria for extracting valid data are:
|
18
|
+
|
19
|
+
**Counting Unique Downloaders**
|
20
|
+
|
21
|
+
downloaders (over a time period like a week) =
|
22
|
+
GET +
|
23
|
+
200 or (206 + (byte range 0-* or -)) +
|
24
|
+
not a bot or filtered IPs) +
|
25
|
+
is > 200,000 bytes +
|
26
|
+
(unique (IP + user agent) or unique (IP + user agent + show))
|
27
|
+
|
28
|
+
|
29
|
+
**Counting Unique Downloads**
|
30
|
+
|
31
|
+
downloads =
|
32
|
+
GET +
|
33
|
+
200 or (206 + (byte range 0-* or -)) +
|
34
|
+
no bots or filtered IPs +
|
35
|
+
is > 200,000 bytes +
|
36
|
+
unique (IP + user agent + file + year + month + day) +
|
37
|
+
byte range start < file size
|
38
|
+
|
39
|
+
|
40
|
+
This gem is primarily intended to be used from the command line, with the output being imported into some other system for further analysis.
|
41
|
+
|
42
|
+
The parser should generally be run over one week of log data as this is typically how radio cume is also measured.
|
43
|
+
|
44
|
+
If you are running this on data from a standalone podcast (no radio broadcasts) it is recommended that you use still a week because
|
45
|
+
the numbers you get will be comparable with the numbers published by public broadcasters who use this methodology.
|
46
|
+
|
47
|
+
## The output
|
48
|
+
|
49
|
+
The script outputs only lines that match according to the Guidelines. It outputs a line of data for each valid log line.
|
50
|
+
|
51
|
+
The first field is the date followed by downloader_uid and download_uid - these are SHA1 hashes of the relevant fields. The full list is:
|
52
|
+
|
53
|
+
date, downloader_uid, download_uid, show_id, status, bytes_sent, url
|
54
|
+
|
55
|
+
We use a SHA1 as this makes subsequent processing for uniqueness simpler.
|
56
|
+
|
57
|
+
## Quick and dirty counts
|
58
|
+
|
59
|
+
To get the number of downloaders:
|
60
|
+
|
61
|
+
prpm-parse-logfile --log-type=nginx podcast.radionz.co.nz-2016-01-31 | cut -f 2 -d ',' | sort | uniq | wc -l
|
62
|
+
|
63
|
+
To get the number of downloads:
|
64
|
+
|
65
|
+
prpm-parse-logfile --log-type=nginx podcast.radionz.co.nz-2016-01-31 | cut -f 3 -d ',' | sort | uniq | wc -l
|
66
|
+
|
67
|
+
You can also pipe the output of gunzip into the script:
|
68
|
+
|
69
|
+
gunzip -c podcast.radionz.co.nz-2016-01-31.gz | prpm-parse-logfile --log-type=nginx | cut -f 2 -d ',' | sort | uniq | wc -l
|
70
|
+
|
71
|
+
|
72
|
+
## Testing the code
|
73
|
+
|
74
|
+
Clone the repository locally, switch to the project root, and run from the terminal like this:
|
75
|
+
|
76
|
+
./bin/prpm-parse-logfile
|
77
|
+
|
78
|
+
## Usage
|
79
|
+
|
80
|
+
TODO: Write full usage instructions
|
81
|
+
|
82
|
+
## Modules
|
83
|
+
|
84
|
+
The gem comes with a number of classes that can be wired together to parse logs
|
85
|
+
|
86
|
+
**Log Line Parser**
|
87
|
+
|
88
|
+
These extract data from one line of a log file. They should be set to only extract HTTP GET requests with status codes 200 or 206
|
89
|
+
|
90
|
+
**Show Filter**
|
91
|
+
|
92
|
+
A show filter is used to parse out the name (or code) for a show from the URL.
|
93
|
+
|
94
|
+
**Bot Filter**
|
95
|
+
|
96
|
+
This checks the user agent string against know bots.
|
97
|
+
(this is a work in progress)
|
98
|
+
|
99
|
+
This command can be used to update the data being used by the gem
|
100
|
+
|
101
|
+
./bin/prpm-update-botdata
|
102
|
+
|
103
|
+
**Line Filter**
|
104
|
+
|
105
|
+
This is where the testing of lines is done against the guidlines.
|
106
|
+
|
107
|
+
NB: For log types that DO NOT contain a range, the filter allows ALL 206 requests for a file through.
|
108
|
+
Because they all have the same downloader_uid and download_uid, they will be filtered later.
|
109
|
+
|
110
|
+
## Contributing
|
111
|
+
|
112
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/radionewzealand/prpm-tools.
|
data/Rakefile
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require 'optparse'
|
5
|
+
require 'ostruct'
|
6
|
+
require "prpmtools"
|
7
|
+
|
8
|
+
include PrpmTools
|
9
|
+
|
10
|
+
# the default options
|
11
|
+
options = OpenStruct.new
|
12
|
+
options.directory = ''
|
13
|
+
options.log_type = nil
|
14
|
+
options.cleanup = false
|
15
|
+
|
16
|
+
ARGV.options do |o|
|
17
|
+
script_name = File.basename($0)
|
18
|
+
|
19
|
+
o.set_summary_indent(' ')
|
20
|
+
o.banner = "Usage: #{script_name} directory_name [OPTIONS]"
|
21
|
+
o.define_head "Scan a specified directory for logs and process them"
|
22
|
+
o.separator ""
|
23
|
+
|
24
|
+
o.on("-v", "--log-type=",
|
25
|
+
"Select the format of the log file",
|
26
|
+
"Default: #{options.log_type}") { |log_type| options.log_type = log_type }
|
27
|
+
|
28
|
+
o.on("-c", "--cleanup",
|
29
|
+
"Remove files after processing",
|
30
|
+
"Default: #{options.cleanup}") { |cleanup| options.cleanup = cleanup }
|
31
|
+
|
32
|
+
o.separator ""
|
33
|
+
|
34
|
+
o.on_tail("-h", "--help", "Show this help message.") { puts o; exit }
|
35
|
+
|
36
|
+
o.parse!
|
37
|
+
end
|
38
|
+
|
39
|
+
date_format = '%d/%b/%Y:%H:%M:%S %z'
|
40
|
+
|
41
|
+
case options.log_type
|
42
|
+
when nil
|
43
|
+
puts 'Enter a log type'
|
44
|
+
exit
|
45
|
+
when 'nginx'
|
46
|
+
line_parser = NginxLogLineParser
|
47
|
+
when 'triton'
|
48
|
+
line_parser = TritonLogLineParser
|
49
|
+
when 'akamai'
|
50
|
+
line_parser = AkamaiLogLineParser
|
51
|
+
date_format = '%Y-%m-%d %H:%M:%S'
|
52
|
+
else
|
53
|
+
puts 'Enter a valid log type'
|
54
|
+
exit
|
55
|
+
end
|
56
|
+
|
57
|
+
ARGF.each do |line|
|
58
|
+
if data = line_parser.new(line).parse
|
59
|
+
if data = LineFilter.new(BotFilter.new, RnzShowParser.new).filter(data, date_format)
|
60
|
+
puts %Q[#{data[:date]},#{data[:downloader_uid]},#{data[:download_uid]},#{data[:show_code]},#{data[:status]},#{data[:bytes_sent]},#{data[:url]}]
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "prpmtools"
|
5
|
+
require 'nokogiri'
|
6
|
+
require 'open-uri'
|
7
|
+
require 'json'
|
8
|
+
|
9
|
+
html_data = open('http://www.useragentstring.com/pages/Crawlerlist/').read
|
10
|
+
nokogiri_object = Nokogiri::HTML(html_data)
|
11
|
+
elements = nokogiri_object.xpath("//h3")
|
12
|
+
|
13
|
+
data = elements.map{ |el| el.text }
|
14
|
+
|
15
|
+
data_file_path = File.expand_path(File.join(File.dirname(File.expand_path(__FILE__)),'..','lib','bot_filter','bot_data.json'))
|
16
|
+
|
17
|
+
puts 'Writing bot data to: ' + data_file_path
|
18
|
+
|
19
|
+
File.open(data_file_path, 'w') do |file|
|
20
|
+
file.write(JSON.generate(data))
|
21
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
["008","ABACHOBot","Accoona-AI-Agent","AddSugarSpiderBot","AnyApexBot","Arachmo","B-l-i-t-z-B-O-T","Baiduspider","BecomeBot","BeslistBot","BillyBobBot","Bimbot","Bingbot","BlitzBOT","boitho.com-dc","boitho.com-robot","btbot","CatchBot","Cerberian Drtrs","Charlotte","ConveraCrawler","cosmos","Covario IDS","DataparkSearch","DiamondBot","Discobot","Dotbot","EmeraldShield.com WebBot","envolk[ITS]spider","EsperanzaBot","Exabot","FAST Enterprise Crawler","FAST-WebCrawler","FDSE robot","FindLinks","FurlBot","FyberSpider","g2crawler","Gaisbot","GalaxyBot","genieBot","Gigabot","Girafabot","Googlebot","Googlebot-Image","GurujiBot","HappyFunBot","hl_ftien_spider","Holmes","htdig","iaskspider","ia_archiver","iCCrawler","ichiro","igdeSpyder","IRLbot","IssueCrawler","Jaxified Bot","Jyxobot","KoepaBot","L.webis","LapozzBot","Larbin","LDSpider","LexxeBot","Linguee Bot","LinkWalker","lmspider","lwp-trivial","mabontland","magpie-crawler","Mediapartners-Google","MJ12bot","Mnogosearch","mogimogi","MojeekBot","Moreoverbot","Morning Paper","msnbot","MSRBot","MVAClient","mxbot","NetResearchServer","NetSeer Crawler","NewsGator","NG-Search","nicebot","noxtrumbot","Nusearch Spider","NutchCVS","Nymesis","obot","oegp","omgilibot","OmniExplorer_Bot","OOZBOT","Orbiter","PageBitesHyperBot","Peew","polybot","Pompos","PostPost","Psbot","PycURL","Qseero","Radian6","RAMPyBot","RufusBot","SandCrawler","SBIder","ScoutJet","Scrubby","SearchSight","Seekbot","semanticdiscovery","Sensis Web Crawler","SEOChat::Bot","SeznamBot","Shim-Crawler","ShopWiki","Shoula robot","silk","Sitebot","Snappy","sogou spider","Sosospider","Speedy Spider","Sqworm","StackRambler","suggybot","SurveyBot","SynooBot","Teoma","TerrawizBot","TheSuBot","Thumbnail.CZ robot","TinEye","truwoGPS","TurnitinBot","TweetedTimes Bot","TwengaBot","updated","Urlfilebot","Vagabondo","VoilaBot","Vortex","voyager","VYU2","webcollage","Websquash.com","wf84","WoFindeIch Robot","WomlpeFactory","Xaldon_WebSpider","yacy","Yahoo! Slurp","Yahoo! Slurp China","YahooSeeker","YahooSeeker-Testing","YandexBot","YandexImages","Yasaklibot","Yeti","YodaoBot","yoogliFetchAgent","YoudaoBot","Zao","Zealbot","zspider","ZyBorg"]
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module PrpmTools
|
4
|
+
|
5
|
+
class BotFilter
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
data_file = File.join(File.dirname(File.expand_path(__FILE__)),'bot_data.json')
|
9
|
+
|
10
|
+
@bots = JSON.parse(File.read(data_file)).join('|')
|
11
|
+
end
|
12
|
+
|
13
|
+
def check_agent(user_agent)
|
14
|
+
return true if user_agent =~ /#{@bots}/
|
15
|
+
false
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
data/lib/line_filter.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'digest'
|
2
|
+
require 'date'
|
3
|
+
|
4
|
+
module PrpmTools
|
5
|
+
|
6
|
+
class LineFilter
|
7
|
+
|
8
|
+
def initialize(bot_filter, show_filter)
|
9
|
+
@bot_filter = bot_filter
|
10
|
+
@show_filter = show_filter
|
11
|
+
end
|
12
|
+
|
13
|
+
def filter(line_data, date_format='%d/%b/%Y:%H:%M:%S %z')
|
14
|
+
convert_line_to_unique_downloader(line_data, date_format)
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def convert_line_to_unique_downloader(data, date_format='%d/%b/%Y:%H:%M:%S %z')
|
20
|
+
|
21
|
+
return if bot_filter(data[:user_agent])
|
22
|
+
return if data[:bytes_sent] < 200000
|
23
|
+
|
24
|
+
# If the log file has NO range (nil) we use a slightly looser match on 206 lines
|
25
|
+
# These will be filtered later through the use of 'unique'
|
26
|
+
if ((data[:status] == 200) || (data[:status] == 206 && (data[:range] =~ /^(0-\d|-)/ || data[:range].nil?) ))
|
27
|
+
data[:date] = DateTime.strptime(data[:timestamp], date_format).to_date.to_s # outputs yyyy-mm-dd
|
28
|
+
data[:downloader_uid] = Digest::SHA1.hexdigest(data[:ip] + data[:user_agent])
|
29
|
+
data[:download_uid] = Digest::SHA1.hexdigest(data[:ip] + data[:user_agent] + data[:url] + data[:date])
|
30
|
+
data[:show_code] = extract_show(data[:url])
|
31
|
+
data
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def bot_filter(user_agent)
|
36
|
+
@bot_filter.check_agent(user_agent)
|
37
|
+
end
|
38
|
+
|
39
|
+
def extract_show(url)
|
40
|
+
@show_filter.extract_show(url)
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module PrpmTools
|
2
|
+
# Log line parsers should only return GET requests and 200 or 206 responses
|
3
|
+
# as these are the only ones of interest to us
|
4
|
+
|
5
|
+
|
6
|
+
# Fields: date time cs-ip cs-method cs-uri sc-status sc-bytes time-taken cs(Referer) cs(User-Agent) cs(Cookie)
|
7
|
+
# 2016-01-31 15:39:05 207.229.180.210 GET /npr.download.akamai.com/npr-mp4/npr/underwriting/042914_AppPermissions_MPX.mp4 200 227434 52 "-" "NPRRadio/161 CFNetwork/711.1.16 Darwin/14.0.0" "-"
|
8
|
+
class AkamaiLogLineParser
|
9
|
+
|
10
|
+
def initialize(line)
|
11
|
+
@line = line
|
12
|
+
end
|
13
|
+
|
14
|
+
def parse
|
15
|
+
%r{
|
16
|
+
(?<date>\d{4}-\d{2}-\d{2})
|
17
|
+
\s+
|
18
|
+
(?<time>\d{2}:\d{2}:\d{2})
|
19
|
+
\s+
|
20
|
+
(?<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})
|
21
|
+
\s+
|
22
|
+
GET
|
23
|
+
\s+
|
24
|
+
(?<url>[^\s]+)
|
25
|
+
\s+
|
26
|
+
(?<status>200|206)
|
27
|
+
\s+
|
28
|
+
(?<bytes_sent>\d+)
|
29
|
+
\s+
|
30
|
+
\d+
|
31
|
+
\s+
|
32
|
+
"[^"]"
|
33
|
+
\s+
|
34
|
+
"(?<user_agent>[^"]+)"
|
35
|
+
\s+
|
36
|
+
"(?<cookie>[^"]+)"
|
37
|
+
}xi =~ @line &&
|
38
|
+
{ip: ip, timestamp: date + ' ' + time , url: url, status: status.to_i, bytes_sent: bytes_sent.to_i, user_agent: user_agent, range: nil, last_byte: nil}
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module PrpmTools
|
2
|
+
# Log line parsers should only return GET requests and 200 or 206 responses
|
3
|
+
# as these are the only ones of interest to us
|
4
|
+
|
5
|
+
class NginxLogLineParser
|
6
|
+
|
7
|
+
def initialize(line)
|
8
|
+
@line = line
|
9
|
+
end
|
10
|
+
|
11
|
+
def parse
|
12
|
+
%r{
|
13
|
+
(?<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})
|
14
|
+
\s+
|
15
|
+
-
|
16
|
+
\s+
|
17
|
+
-
|
18
|
+
\s+
|
19
|
+
\[(?<timestamp>[^\]]+?)\]
|
20
|
+
\s+
|
21
|
+
"GET\s(?<url>[^\s]+?)\s(HTTP\/1\.1)"
|
22
|
+
\s+
|
23
|
+
(?<status>200|206)
|
24
|
+
\s+
|
25
|
+
(?<bytes_sent>\d+)
|
26
|
+
\s+
|
27
|
+
"-"
|
28
|
+
\s+
|
29
|
+
"(?<user_agent>.*)"
|
30
|
+
}xi =~ @line &&
|
31
|
+
{ip: ip, timestamp: timestamp, url: url, status: status.to_i, bytes_sent: bytes_sent.to_i, user_agent: user_agent, range: nil, last_byte: nil}
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module PrpmTools
|
2
|
+
# Log line parsers should only return GET requests and 200 or 206 responses
|
3
|
+
# as these are the only ones of interest to us
|
4
|
+
|
5
|
+
|
6
|
+
#Fields: ip client_id username timestamp request response bytes_sent referer user_agent object_size byte_range last_byte
|
7
|
+
# 162.207.92.164 - "cookie:97819a53-cbad-430d-a068-9e2b77993b8b" [01/Feb/2016:00:04:43 -0500] "GET /NPR_381444908/media-session/8621a9ec-1bfe-4688-82c1-ef0119226ec4/anon.npr-podcasts/podcast/381444908/462702123/npr_462702123.mp3?STW_NEW_UUID=1&d=2864&e=462702123&f=381444908&ft=pod&orgId=1&p=381444908&story=462702123&t=podcast HTTP/1.1" 200 94120 - "AppleCoreMedia/1.0.0.12D508 (iPad; U; CPU OS 8_2 like Mac OS X; en_us)" 23053548 0-23053547 0
|
8
|
+
class TritonLogLineParser
|
9
|
+
|
10
|
+
def initialize(line)
|
11
|
+
@line = line
|
12
|
+
end
|
13
|
+
|
14
|
+
def parse
|
15
|
+
%r{
|
16
|
+
(?<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})
|
17
|
+
\s+
|
18
|
+
-
|
19
|
+
\s+
|
20
|
+
.+
|
21
|
+
\s+
|
22
|
+
\[(?<timestamp>[^\]]+?)\]
|
23
|
+
\s+
|
24
|
+
"GET\s(?<url>[^\s]+?)\s(HTTP\/1\.1)"
|
25
|
+
\s+
|
26
|
+
(?<status>200|206)
|
27
|
+
\s+
|
28
|
+
(?<bytes_sent>\d+)
|
29
|
+
\s+
|
30
|
+
-
|
31
|
+
\s+
|
32
|
+
"(?<user_agent>.*)"
|
33
|
+
\s+
|
34
|
+
(?<total_bytes>\d+)
|
35
|
+
\s+
|
36
|
+
(?<range>(\d+-\d+|-))
|
37
|
+
\s+
|
38
|
+
(?<last_byte>\d)
|
39
|
+
}xi =~ @line &&
|
40
|
+
{ip: ip, timestamp: timestamp, url: url, status: status.to_i, bytes_sent: bytes_sent.to_i, user_agent: user_agent, total_bytes: total_bytes, range: range, last_byte: last_byte}
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
data/lib/prpmtools.rb
ADDED
data/license.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2016 Richard Hulse/Radio New Zealand Limited
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/prpmtools.gemspec
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'prpmtools/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "prpm-tools"
|
8
|
+
spec.version = PrpmTools::VERSION
|
9
|
+
spec.authors = ["Richard Hulse"]
|
10
|
+
spec.email = ["richard.hulse@radionz.co.nz"]
|
11
|
+
|
12
|
+
spec.summary = %q{Tools for parsing logs according the PRPM Guidlines}
|
13
|
+
spec.homepage = "https://github.com/radionewzealand/prpm-tools"
|
14
|
+
|
15
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
16
|
+
spec.bindir = "exe"
|
17
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
18
|
+
spec.require_paths = ["lib"]
|
19
|
+
|
20
|
+
spec.add_development_dependency "bundler", "~> 1.10"
|
21
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
22
|
+
spec.add_development_dependency "rspec"
|
23
|
+
|
24
|
+
spec.add_dependency "nokogiri"
|
25
|
+
|
26
|
+
end
|
metadata
ADDED
@@ -0,0 +1,118 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: prpm-tools
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Richard Hulse
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-03-04 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.10'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.10'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: nokogiri
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description:
|
70
|
+
email:
|
71
|
+
- richard.hulse@radionz.co.nz
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- ".gitignore"
|
77
|
+
- ".rspec"
|
78
|
+
- ".travis.yml"
|
79
|
+
- Gemfile
|
80
|
+
- README.md
|
81
|
+
- Rakefile
|
82
|
+
- bin/prpm-parse-logfile
|
83
|
+
- bin/prpm-update-botdata
|
84
|
+
- lib/bot_filter/bot_data.json
|
85
|
+
- lib/bot_filter/bot_filter.rb
|
86
|
+
- lib/line_filter.rb
|
87
|
+
- lib/log_line_parsers/akamai.rb
|
88
|
+
- lib/log_line_parsers/nginx.rb
|
89
|
+
- lib/log_line_parsers/triton.rb
|
90
|
+
- lib/prpmtools.rb
|
91
|
+
- lib/prpmtools/version.rb
|
92
|
+
- lib/show_parsers/rnz.rb
|
93
|
+
- license.txt
|
94
|
+
- prpmtools.gemspec
|
95
|
+
homepage: https://github.com/radionewzealand/prpm-tools
|
96
|
+
licenses: []
|
97
|
+
metadata: {}
|
98
|
+
post_install_message:
|
99
|
+
rdoc_options: []
|
100
|
+
require_paths:
|
101
|
+
- lib
|
102
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
103
|
+
requirements:
|
104
|
+
- - ">="
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: '0'
|
107
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
108
|
+
requirements:
|
109
|
+
- - ">="
|
110
|
+
- !ruby/object:Gem::Version
|
111
|
+
version: '0'
|
112
|
+
requirements: []
|
113
|
+
rubyforge_project:
|
114
|
+
rubygems_version: 2.4.8
|
115
|
+
signing_key:
|
116
|
+
specification_version: 4
|
117
|
+
summary: Tools for parsing logs according the PRPM Guidlines
|
118
|
+
test_files: []
|