picky-statistics 4.5.0 → 4.5.1

data/lib/picky-statistics/application/stylesheets/statistics.css

@@ -1,110 +1,176 @@
+* { box-sizing: border-box; }
+
 body {
-  font-family: Futura, 'Century Gothic', 'URW Gothic L', Helvetica, sans-serif;
+  font-family: "Helvetica Neue", Helvetica;
+  font-size: 16px;
   background-image: url('../images/background.png');
+  text-rendering: optimizeLegibility;
+  padding: 1em;
+  width: 1024px;
 }
 
-h1, h2, h3, h4 {
-  font-weight: normal;
+h1, h2, h3, h4, aside {
+  font-weight: 100;
+  text-transform: uppercase;
+  letter-spacing: 3px;
   margin: 0.5em 0em 0.5em;
 }
 h1 { font-size: 1.3em; }
-h2 { font-size: 1.2em; }
-h3 { font-size: 1.1em; }
-h4 { font-size: 1.1em; }
+h2 { font-size: 1.1em; }
+h3 { font-size: 1.0em; }
+h4 { font-size: 0.9em; }
 
 input {
   margin-left: 10px;
   margin-right: 5px;
 }
 
+#notice {
+  font-size: 3em;
+  font-weight: 100;
+  text-align: center;
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  opacity: 0.7;
+  background-color: white;
+  background: -moz-radial-gradient(50% 50%, circle, white 150px, lightgray 600px);
+  background: -webkit-gradient(radial, 50% 50%, 150, 50% 70%, 600, from(white), to(lightgray));
+  display: none;
+}
+#notice > div {
+  position: absolute;
+  top: 50%;
+  left: 50%;
+  width: 10em;
+  height: 4em;
+  margin: -2em 0px 0px -5em;
+}
+#notice.show {
+  display: block;
+}
+
 .actions {
   padding: 5px;
   font-size: 0.8em;
 }
 
-.stats {
-  padding: 10px;
-  clear: both;
+.search {
+  font-family: "Courier New", monospace;
+  overflow: hidden;
 }
 
-.rickshaw_legend {
+.search > div {
   float: left;
+  padding: 5px;
 }
 
-.rickshaw_legend ul {
-  padding: 0px;
-  margin: 0px;
+.text, .total, .allocation, .offset, .duration {
+  font-weight: bold;
 }
 
-.rickshaw_legend .line {
-  clear: both;
-  line-height: 140%;
-  padding-right: 15px;
+.search > .time {
+  width: 6em;
 }
 
-.rickshaw_legend li {
-  padding: 0 0 0 2px;
-  min-width: 80px;
-  white-space: nowrap;
-  list-style-type: none;
+.search > .text {
+  width: 22em;
 }
 
-.rickshaw_legend .line .swatch {
+.search > .total {
+  text-align: right;
+  width: 4em;
+}
+
+.search > .offset {
+  text-align: right;
+  width: 2em;
+}
+
+.search > .duration {
+  text-align: right;
+  width: 6em;
+}
+
+span#active, span#total {
+  font-weight: bold;
+}
+
+#body > p {
+  line-height: 1.5em;
+  width: 800px;
+  text-rendering: optimizeLegibility;
+}
+
+#charts {
+  padding: 10px 0;
+}
+
+.chart {
   display: inline-block;
-  margin-right: 3px;
-  border-radius: 2px;
+  height: 151px;
+  margin-bottom: 20px;
 }
 
-.rickshaw_legend .swatch {
-  width: 10px;
-  height: 10px;
-  border: 1px solid rgba(0, 0, 0, 0.2);
+.reset {
+  padding-left: 1em;
+  font-size: smaller;
+  color: #ccc;
 }
 
-.rickshaw_graph {
-  float: left;
-  margin-right: 20px;
+.background.bar {
+  fill: #ccc;
 }
 
-.rickshaw_graph .detail {
-  pointer-events: none;
-  position: relative;
-  top: 0;
-  z-index: 2;
-  background: rgba(0, 0, 0, 0.1);
-  bottom: 0;
-  width: 1px;
-  transition: opacity 0.25s linear;
-  -moz-transition: opacity 0.25s linear;
-  -o-transition: opacity 0.25s linear;
-  -webkit-transition: opacity 0.25s linear;
+.foreground.bar {
+  fill: steelblue;
 }
 
-.rickshaw_graph .detail .item {
-  position: absolute;
-  z-index: 2;
-  border-radius: 3px;
-  padding: 0.25em;
-  font-size: 12px;
-  font-family: Arial, sans-serif;
-  opacity: 0;
-  background: rgba(0, 0, 0, 0.4);
-  color: white;
-  border: 1px solid rgba(0, 0, 0, 0.4);
-  margin-left: 1em;
-  margin-top: -1em;
-  white-space: nowrap;
-}
-
-
-.rickshaw_graph .detail .x_label {
-  font-family: Arial, sans-serif;
-  border-radius: 3px;
-  padding: 6px;
-  opacity: 0.5;
-  border: 1px solid #E0E0E0;
-  font-size: 12px;
-  position: absolute;
-  background: white;
-  white-space: nowrap;
+.axis path, .axis line {
+  fill: none;
+  stroke: #000;
+  shape-rendering: crispEdges;
+}
+
+.axis text {
+  font: 10px sans-serif;
+}
+
+.brush rect.extent {
+  fill: steelblue;
+  fill-opacity: .125;
+}
+
+.brush .resize path {
+  fill: #eee;
+  stroke: #666;
+}
+
+#search-list {
+  min-height: 1024px;
+}
+
+#search-list .date,
+#search-list .day {
+  margin-bottom: .4em;
+}
+
+#search-list .search {
+  line-height: 1.5em;
+  background: #eee;
+  margin-bottom: 1px;
 }
+
+#search-list .time {
+  color: #999;
+}
+
+#search-list .early {
+  color: green;
+}
+
+aside {
+  float: right;
+}

data/lib/picky-statistics/application/views/index.haml

@@ -2,41 +2,50 @@
 %html{ :lang => 'en' }
   %head
     %link{ :href => 'stylesheets/statistics.css', :media => 'screen', :rel => 'stylesheet', :type => 'text/css' }/
-    %script{ :src => 'javascripts/jquery-1.4.4.min.js', :type => 'text/javascript' }
-    %script{ :src => 'javascripts/jquery.sparkline-1.6.min.js', :type => 'text/javascript' }
-    %script{ :src => 'javascripts/d3/d3.min.js', :type => 'text/javascript' }
-    %script{ :src => 'javascripts/d3/d3.layout.min.js', :type => 'text/javascript' }
-    %script{ :src => 'javascripts/rickshaw.js', :type => 'text/javascript' }
-    %script{ :src => 'javascripts/graph.js', :type => 'text/javascript' }
+    %script{ :src => 'javascripts/d3.v2.min.js', :type => 'text/javascript' }
+    %script{ :src => 'javascripts/crossfilter.v1.min.js', :type => 'text/javascript' }
+    %script{ :src => 'javascripts/helpers.js', :type => 'text/javascript' }
+    %script{ :src => 'javascripts/statistics.js', :type => 'text/javascript' }
 
-  %body
+  %body{ :onload => "document.getElementsByTagName('input')[0].click(); updateStatistics(true);" }
     %h1 Clam: Picky Statistics Interface.
     .actions
       %button{ :href => '#', :onclick => 'javascript:updateStatistics();' } Update now
+      %button{ :href => '#', :onclick => "javascript:document.getElementsByTagName('input')[0].click();" } Stop Updating
       |
-      %input{ :type => "radio", :name => "updating", :onclick => 'javascript:stopUpdatingStatistics();' } Don't update
-      %input{ :type => "radio", :name => "updating", :onclick => 'javascript:updateStatisticsPeriodically(1);' } Every second
-      %input{ :type => "radio", :name => "updating", :onclick => 'javascript:updateStatisticsPeriodically(3);', :checked => 'checked' } Every 3 seconds
-      %input{ :type => "radio", :name => "updating", :onclick => 'javascript:updateStatisticsPeriodically(10);' } Every 10 seconds
-      %input{ :type => "radio", :name => "updating", :onclick => 'javascript:updateStatisticsPeriodically(30);'} Every 30 seconds
-      %input{ :type => "radio", :name => "updating", :onclick => 'javascript:updateStatisticsPeriodically(600);'} Every 5 minutes
-    #full_live_graph.stats
-      %h3 Full/Live
-      .graph
-      .legend
-    #results_graph.stats
-      %h3 Full Results
-      .graph
-      .legend
-    #speed_graph.stats
-      %h3 Full Performance
-      .graph
-      .legend
-    #offset_graph.stats
-      %h3 Full Offsets
-      .graph
-      .legend
-    %script{ :src => 'javascripts/statistics.js', :type => 'text/javascript' }
-  :javascript
-    updateStatistics();
-    updateStatisticsPeriodically(3);
+      %input{ :id => 'updateStop', :type => "radio", :name => "updating", :onclick => 'javascript:stopUpdatingStatistics();' }
+      %label{ :for => 'updateStop' } Don't update
+      %input{ :id => 'update3', :type => "radio", :name => "updating", :onclick => 'javascript:updateStatisticsPeriodically(3);' }
+      %label{ :for => 'update3' } Every 3 seconds
+      %input{ :id => 'update10', :type => "radio", :name => "updating", :onclick => 'javascript:updateStatisticsPeriodically(10);' }
+      %label{ :for => 'update10' } Every 10 seconds
+      %input{ :id => 'update30', :type => "radio", :name => "updating", :onclick => 'javascript:updateStatisticsPeriodically(30);'}
+      %label{ :for => 'update30' } Every 30 seconds
+      %input{ :id => 'update600', :type => "radio", :name => "updating", :onclick => 'javascript:updateStatisticsPeriodically(600);'}
+      %label{ :for => 'update600' } Every 5 minutes
+
+#notice
+  %div READING LOG&nbspFILE
+
+#charts
+  #hour-chart.chart
+    %h2.title Time of Day
+  #total-chart.chart
+    %h2.title Total
+  #allocations-chart.chart
+    %h2.title Allocations
+  #offset-chart.chart
+    %h2.title Offset
+  #duration-chart.chart
+    %h2.title Duration
+
+%aside#totals
+  %span#active
+    -
+  of
+  %span#total
+    -
+  searches selected.
+
+#lists
+  #search-list.list

data/lib/picky-statistics/logfile_reader.rb

@@ -0,0 +1,106 @@
+require 'tempfile'
+require 'json'
+
+class LogfileReader
+
+  attr_reader :last_offset # in bytes
+  attr_writer :path
+
+  def exclaim text
+    puts text
+  end
+
+  #
+  #
+  def initialize path
+    @path = File.expand_path path
+    check_file
+  end
+  def check_file
+    if File.exists? @path
+      exclaim "Logfile #{@path} found."
+    else
+      raise "Log file #{@path} not found."
+    end
+  end
+  
+  def reset
+    @last_offset = 0
+  end
+
+  def since log_offset = 0
+    @last_offset ||= 0
+    
+    start_time = Time.now
+
+    # Add all the data to the results.
+    #
+    results = []
+    with_temp_file(log_offset) do |statistics|
+      calculate_last_offset_from statistics
+      
+      File.open(statistics, 'r') do |file|
+        while line = file.gets
+          next if line =~ /\A\s*\#/
+          results << process(line)
+        end
+      end
+    end
+
+    duration = Time.now - start_time
+    exclaim "Parsed log from byte #{log_offset} in #{duration}s"
+      
+    results
+  end
+
+  #
+  #
+  def since_last
+    since @last_offset
+  end
+  
+  # Processes one line and returns an array.
+  #
+  def process line
+    [
+      line[2,  19],
+      Float(line[22, 8]),
+      line[31, 50].gsub!(/\s*\z/, ''),
+      Integer(line[82, 8]),
+      Integer(line[92, 3]),
+      Integer(line[96, 2])
+    ]
+  end
+
+  def calculate_last_offset_from statistics
+    @last_offset += last_offset_from statistics
+  end
+  def last_offset_from statistics
+    `wc #{statistics}`.split(/\s+/)[3].to_i
+  end
+
+  #
+  #
+  def reset_from time
+    with_temp_file(offset) do |statistics|
+      full[:total].reset_from statistics
+    end
+    @counts
+  end
+
+  # Use the offset to speed up statistics gathering.
+  #
+  def with_temp_file offset = 0
+    # Quickly return if no logs have been written since the last time.
+    #
+    return if last_offset_from(@path) <= last_offset
+
+    Tempfile.open 'picky' do |temp_file|
+      temp_path = temp_file.path
+      `tail -c +#{offset} #{@path} > #{temp_path}`
+        
+      yield temp_path
+    end
+  end
+
+end

data/spec/lib/picky-statistics/logfile_reader_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+describe LogfileReader do
+
+  let(:reader) do
+    reader = LogfileReader.allocate
+    reader.stub! :exclaim
+    reader.send :initialize, 'spec/data/search.log'
+    reader
+  end
+  
+  describe 'process' do
+    it 'works correctly' do
+      reader.process(
+        ">|2010-12-12 19:36:33|0.000197|s                                                 |10000000|   0| 1|"
+      ).should == [
+        '2010-12-12 19:36:33',
+        0.000197,
+        's',
+        10000000,
+        0,
+        1
+      ]
+    end
+  end
+  
+  describe 'since_last' do
+    it 'works correctly' do
+      reader.since(0).should == [
+        ["2010-12-09 16:13:45", 1.0e-06,  "", 0, 0, 0],
+        ["2010-12-09 16:13:45", 1.0e-05,  "", 1, 0, 1],
+        ["2010-12-09 16:13:45", 0.0001,   "", 2, 0, 2],
+        ["2010-12-12 12:42:19", 0.000999, "", 3, 0, 3],
+        ["2010-12-12 12:42:19", 0.001,    "", 4, 20, 4],
+        ["2010-12-12 12:42:19", 0.009999, "", 5, 40, 5],
+        ["2010-12-12 17:20:55", 0.01,     "", 10, 0, 1],
+        ["2010-12-12 17:20:55", 0.099999, "", 15, 0, 1],
+        ["2010-12-12 17:20:55", 0.1,      "", 20, 0, 1],
+        ["2010-12-12 17:21:32", 0.999999, "", 25, 0, 1],
+        ["2010-12-12 17:21:32", 1.0,      "", 100, 0, 1],
+        ["2010-12-12 17:21:32", 9.999999, "", 500, 0, 1],
+        ["2010-12-12 18:06:24", 6.5e-05,  "", 1000, 0, 1],
+        ["2010-12-12 18:06:24", 4.5e-05,  "", 5000, 0, 1],
+        ["2010-12-12 18:06:24", 1.8e-05,  "", 10000, 0, 10],
+        ["2010-12-12 19:36:00", 0.001109, "s", 50000, 0, 1],
+        ["2010-12-12 19:36:24", 0.000199, "s", 100000, 0, 1],
+        ["2010-12-12 19:36:32", 0.00016,  "s", 500000, 0, 1],
+        ["2010-12-12 19:36:32", 0.001147, "s", 1000000, 0, 1],
+        ["2010-12-12 19:36:33", 0.000209, "s", 5000000, 0, 1],
+        ["2010-12-12 19:36:33", 0.000197, "s", 10000000, 0, 1],
+        ["2010-12-12 19:36:33", 0.000188, "s", 99999999, 0, 99],
+        ["2010-12-09 19:45:01", 1.0e-06,  "", 0, 0, 0]
+      ]
+    end
+  end
+
+end