wikiranger 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 52617871050938bbf1b5db6723a178e82ebffb0b
-  data.tar.gz: d1a3c7941a801cb21214b29d0c565fbfc86decfc
+  metadata.gz: ee56f0d0a12d1f38a39c3b63b17408da3d0870b9
+  data.tar.gz: 9e2ae0ebae414dd022dfe9f14ddf786494e4a256
 SHA512:
-  metadata.gz: 27942efad962fae9cef47deb31f3ee55833eaabf6ede6ef6c841add21f585179b499349734b11758b57997bd741289e69f1371456cfa2e0e11ab03e9c5e9fbc8
-  data.tar.gz: 0e51333984e97e4155671614e59da774a8fe5013faef746ece51ebdb911a8c1030633ef59e3297ff0869acbe24100c38ff437a06d081a53d8729d2d877cb743d
+  metadata.gz: 3cff48a388b7bcc366394c62969d2223d1593f05edd6088b8e6011579ebf1fa583f75c12e98369fd692b293364ab6fc27c592af42be72996107d72b515713ef0
+  data.tar.gz: ee3a860d27cd930481df8786432cd3292d1f79a7799e6bf9f0597b7ba1e7d94c1cc3512250fe302eb522742170f27a126f21962ed11724e55db92a637e57b2e6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    wikiranger (0.1.0)
+    wikiranger (0.2.0)
       colorize (~> 0.8.1)
       ruby-progressbar (~> 1.9)
 

data/exe/wikiranger

@@ -4,6 +4,7 @@ require "wikiranger"
 
 options = {
   :csv          => nil,
+  :html         => nil,
   :threads      => 5,
   :top          => 10,
   :api_base_url => Wikiranger::Wikipedia::DEFAULT_API_BASE_URI
@@ -13,6 +14,14 @@ begin
   OptionParser.new do |opts|
     opts.banner = "Usage: #{$0} [options] cidr [cidr2] ... [cidrN]"
 
+    opts.on("--html DESTINATION", "Write Wiki contribution HTML report file") do |v|
+      if File.exists?(v)
+        puts "ERROR: File #{v} does already exist".red
+        exit 1
+      end
+      options[:html] = v
+    end
+
     opts.on("--csv DESTINATION", "Write Wiki contribution data to CSV file") do |v|
       if File.exists?(v)
         puts "ERROR: File #{v} does already exist".red
@@ -97,9 +106,15 @@ if contributions.count.zero?
   exit
 end
 
+if options[:html]
+  report = Wikiranger::Report.new(contributions)
+  report.generate(options[:html])
+  puts "[+] Wrote Wiki contribution HTML report to #{options[:html].bold}"
+end
+
 if options[:csv]
   CSV.open(options[:csv], "wb") do |csv|
-    csv << ["user", "page_id", "rev_id", "parent_id", "title", "timestamp", "comment", "size"]
+    csv << ["user", "page_id", "rev_id", "parent_id", "title", "timestamp", "comment", "size", "size_diff"]
     contributions.each do |contribution|
       csv << contribution.to_csv_array
     end
@@ -120,10 +135,10 @@ contributions.each do |contribution|
   end
 
   if !top_pages.key?(contribution.pageid)
-    top_pages[contribution.pageid] = [contribution.size, contribution.title]
+    top_pages[contribution.pageid] = [contribution.sizediff.abs, contribution.title]
   else
     contribution_bytes, title = top_pages[contribution.pageid]
-    top_pages[contribution.pageid] = [contribution_bytes + contribution.size, title]
+    top_pages[contribution.pageid] = [contribution_bytes + contribution.sizediff.abs, title]
   end
 end
 

data/lib/wikiranger.rb

@@ -6,6 +6,8 @@ require "optparse"
 require "ipaddr"
 require "date"
 require "csv"
+require "erb"
+require "cgi"
 
 require "colorize"
 require "ruby-progressbar"
@@ -15,8 +17,11 @@ require "wikiranger/util"
 require "wikiranger/thread_pool"
 require "wikiranger/wikipedia"
 require "wikiranger/wikipedia/user_contribution"
+require "wikiranger/report"
 
 module Wikiranger
+  WIKIRANGER_ROOT = File.expand_path(File.join(File.dirname(__FILE__), "..")).freeze
+
   BANNER = "       _ _   _\n" +
            " _ _ _|_| |_|_|___ ___ ___ ___ ___ ___\n" +
            "| | | | | '_| |  _| .'|   | . | -_|  _|\n" +

data/lib/wikiranger/report.rb

@@ -0,0 +1,45 @@
+module Wikiranger
+  class Report
+    def initialize(contributions)
+      @contributions = contributions.sort_by { |c| c.timestamp_object }.reverse
+      @contributors  = contributions.map(&:user).uniq.sort_by { |ip| ip.split(".").map(&:to_i) }
+      @pages         = contributions.map { |c| [c.pageid, c.title] }.uniq.sort_by { |p| p.last }
+      @contribs_per_contributor = @contributors.map do |c|
+        [c, contributions.count { |contrib| contrib.user == c }]
+      end.to_h
+      @contribs_per_page = @pages.map do |p|
+        [p.first, contributions.count { |contrib| contrib.pageid == p.first }]
+      end.to_h
+    end
+
+    def generate(destination)
+      report = load_template
+      b      = binding
+      File.open(destination, "w") do |f|
+        f.write(report.result(b))
+      end
+    end
+
+    private
+
+    def load_template
+      ERB.new(File.read(File.join(Wikiranger::WIKIRANGER_ROOT, "templates", "default.html.erb")))
+    end
+
+    def h(unsafe)
+      CGI.escapeHTML(unsafe.to_s)
+    end
+
+    def contributor_url(contribution)
+      "https://en.m.wikipedia.org/wiki/Special:Contributions/#{h(contribution.user)}"
+    end
+
+    def contribution_url(contribution)
+      "https://en.m.wikipedia.org/wiki/Special:MobileDiff/#{h(contribution.revid)}"
+    end
+
+    def page_url(contribution)
+      "https://en.wikipedia.org/?curid=#{h(contribution.pageid)}"
+    end
+  end
+end

data/lib/wikiranger/version.rb

@@ -1,3 +1,3 @@
 module Wikiranger
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/wikiranger/wikipedia.rb

@@ -12,7 +12,7 @@ module Wikiranger
     end
 
     def user_contributions(user)
-      uri      = "#{api_base_uri}/w/api.php?action=query&format=json&list=usercontribs&uclimit=max&ucuser=#{URI.escape(user)}&ucdir=older"
+      uri      = "#{api_base_uri}/w/api.php?action=query&format=json&list=usercontribs&uclimit=max&ucuser=#{URI.escape(user)}&ucdir=older&ucprop=&ucprop=ids|title|timestamp|comment|size|sizediff"
       response = request(uri)
       if response.code.to_i != 200
         unexpected_response!("Unexpected response code: #{response.code} when retrieving user contributions for #{user}")

data/lib/wikiranger/wikipedia/user_contribution.rb

@@ -13,7 +13,7 @@ module Wikiranger
       end
 
       def to_csv_array
-        [self.user, self.pageid, self.revid, self.parentid, self.title, self.timestamp, self.comment, self.size]
+        [self.user, self.pageid, self.revid, self.parentid, self.title, self.timestamp, self.comment, self.size, self.sizediff]
       end
     end
   end

data/templates/default.html.erb

@@ -0,0 +1,181 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Wikiranger HTML Report</title>
+    <style type="text/css">
+      /*! normalize.css v7.0.0 | MIT License | github.com/necolas/normalize.css */html{line-height:1.15;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}article,aside,footer,header,nav,section{display:block}h1{font-size:2em;margin:.67em 0}figcaption,figure,main{display:block}figure{margin:1em 40px}hr{box-sizing:content-box;height:0;overflow:visible}pre{font-family:monospace,monospace;font-size:1em}a{background-color:transparent;-webkit-text-decoration-skip:objects}abbr[title]{border-bottom:0;text-decoration:underline;text-decoration:underline dotted}b,strong{font-weight:inherit}b,strong{font-weight:bolder}code,kbd,samp{font-family:monospace,monospace;font-size:1em}dfn{font-style:italic}mark{background-color:#ff0;color:#000}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-0.25em}sup{top:-0.5em}audio,video{display:inline-block}audio:not([controls]){display:none;height:0}img{border-style:none}svg:not(:root){overflow:hidden}button,input,optgroup,select,textarea{font-family:sans-serif;font-size:100%;line-height:1.15;margin:0}button,input{overflow:visible}button,select{text-transform:none}button,html [type="button"],[type="reset"],[type="submit"]{-webkit-appearance:button}button::-moz-focus-inner,[type="button"]::-moz-focus-inner,[type="reset"]::-moz-focus-inner,[type="submit"]::-moz-focus-inner{border-style:none;padding:0}button:-moz-focusring,[type="button"]:-moz-focusring,[type="reset"]:-moz-focusring,[type="submit"]:-moz-focusring{outline:1px dotted ButtonText}fieldset{padding:.35em .75em .625em}legend{box-sizing:border-box;color:inherit;display:table;max-width:100%;padding:0;white-space:normal}progress{display:inline-block;vertical-align:baseline}textarea{overflow:auto}[type="checkbox"],[type="radio"]{box-sizing:border-box;padding:0}[type="number"]::-webkit-inner-spin-button,[type="number"]::-webkit-outer-spin-button{height:auto}[type="search"]{-webkit-appearance:textfield;outline-offset:-2px}[type="search"]::-webkit-search-cancel-button,[type="search"]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}details,menu{display:block}summary{display:list-item}canvas{display:inline-block}template{display:none}[hidden]{display:none}
+
+      body {
+        font-family: Atlas Grotesk Web,helvetica neue,helvetica,arial,sans-serif;
+        color: #494a4b;
+        background-color: #f5f5f5;
+        -webkit-font-smoothing: antialiased;
+        -moz-osx-font-smoothing: grayscale;
+        overflow-y: scroll;
+        padding-bottom: 50px;
+      }
+
+      h1,h2,h3,h4,h5,h6 {
+        color: #494a4b;
+        font-weight: 400;
+      }
+
+      p {
+        margin:0 0 1em 0;
+        line-height:1.5;
+        font-size:17px;
+      }
+
+      a {
+        color: inherit;
+        text-decoration: underline;
+      }
+
+      a:hover {
+        color: #000;
+      }
+
+      fieldset {
+        margin-bottom: 20px;
+      }
+
+      legend {
+        font-weight: bold;
+        padding-left: 5px;
+        padding-right: 5px;
+      }
+
+      .hidden {
+        display: none;
+      }
+
+      .content {
+        padding: 0px 20px 0px 20px;
+      }
+
+      .logo {
+        background-color: #2a2730;
+        color: #99979c;
+        margin: 0px 0px 50px 0px;
+        font-weight: bold;
+      }
+
+      .logo a {
+        color: #99979c;
+      }
+
+      .logo weak {
+        font-weight: normal;
+      }
+
+      table#contributions th {
+        text-align: left;
+      }
+
+      table#contributions tbody tr:hover {
+        background-color: #eaeaea;
+      }
+
+      .col-time {
+        width: 150px;
+      }
+
+      .diff-remove {
+        color: #dc2e16;
+      }
+
+      .diff-add {
+        color: #00af88;
+      }
+    </style>
+  </head>
+  <body>
+<pre class="logo">
+<div class="content">
+       _ _   _
+ _ _ _|_| |_|_|___ ___ ___ ___ ___ ___
+| | | | | '_| |  _| .'|   | . | -_|  _|
+|_____|_|_,_|_|_| |__,|_|_|_  |___|_| v<%= Wikiranger::VERSION %>
+         <weak>by <a href="https://twitter.com/michenriksen" target="_blank">@michenriksen</a></weak> |___|
+
+</div>
+</pre>
+    <div class="content">
+      <fieldset>
+        <legend>Filtering options</legend>
+        <select id="select_contributor">
+          <option value="" selected="selected">From all contributors</option>
+          <% @contributors.each do |contributor| %>
+            <option value="<%=h contributor %>"><%=h contributor %> (<%=h @contribs_per_contributor[contributor] || '0' %>)</option>
+          <% end %>
+        </select>
+        <select id="select_page">
+          <option value="" selected="selected">From all pages</option>
+          <% @pages.each do |page| %>
+            <option value="<%=h page.first %>"><%=h page.last %> (<%=h @contribs_per_page[page.first] || '0' %>)</option>
+          <% end %>
+        </select>
+        <select id="select_changetype">
+          <option value="" selected="selected">Removals and additions</option>
+          <option value="removal">Removals</option>
+          <option value="addition">Additions</option>
+        </select>
+      </fieldset>
+      <table cellspacing="0" id="contributions" class="contribution-table">
+        <thead>
+          <tr>
+            <th class="col-time">Time</th>
+            <th>Change</th>
+            <th>Contributor</th>
+            <th>Page</th>
+            <th>Comment</th>
+          </tr>
+        </thead>
+        <tbody>
+          <% @contributions.each do |contribution| %>
+            <tr data-user="<%=h contribution.user %>" data-page-id="<%=h contribution.pageid %>" data-change-type="<%=h contribution.sizediff < 0 ? 'removal' : 'addition' %>">
+              <td><%=h contribution.timestamp_object.strftime("%b %e %Y, %H:%M") %></td>
+              <td class="<%= contribution.sizediff < 0 ? 'diff-remove' : 'diff-add' %>"><%=h contribution.sizediff%></td>
+              <td><a href="<%= contributor_url(contribution) %>" target="_blank"><%=h contribution.user %></a></td>
+              <td><a href="<%= page_url(contribution) %>" target="_blank"><%=h contribution.title %></a></td>
+              <td><a href="<%= contribution_url(contribution) %>" target="_blank"><%=h contribution.comment.empty? ? 'no edit summary' : contribution.comment %></a></td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+    </div>
+    <script type="text/javascript">
+      function filterContributions() {
+        var contributor   = document.getElementById("select_contributor").value;
+        var page          = document.getElementById("select_page").value;
+        var changeType    = document.getElementById("select_changetype").value;
+        var contributions = document.querySelectorAll('#contributions tbody tr');
+
+        Array.prototype.forEach.call(contributions, function(el, i) {
+          el.classList.remove("hidden");
+          if (contributor !== "" && el.getAttribute("data-user") !== contributor ) {
+            el.classList.add("hidden");
+            return;
+          }
+
+          if (page !== "" && el.getAttribute("data-page-id") !== page) {
+            el.classList.add("hidden");
+            return;
+          }
+
+          if (changeType !== "" && el.getAttribute("data-change-type") !== changeType) {
+            el.classList.add("hidden");
+            return;
+          }
+        });
+      }
+
+      document.getElementById("select_contributor").addEventListener("change", filterContributions);
+      document.getElementById("select_page").addEventListener("change", filterContributions);
+      document.getElementById("select_changetype").addEventListener("change", filterContributions);
+    </script>
+  </body>
+</html>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wikiranger
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Michael Henriksen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-14 00:00:00.000000000 Z
+date: 2018-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
@@ -99,11 +99,13 @@ files:
 - bin/setup
 - exe/wikiranger
 - lib/wikiranger.rb
+- lib/wikiranger/report.rb
 - lib/wikiranger/thread_pool.rb
 - lib/wikiranger/util.rb
 - lib/wikiranger/version.rb
 - lib/wikiranger/wikipedia.rb
 - lib/wikiranger/wikipedia/user_contribution.rb
+- templates/default.html.erb
 - wikiranger.gemspec
 homepage: https://github.com/michenriksen/wikiranger
 licenses: