shades 0.11 → 0.12

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    Y2M3MzYxNDM1Y2VlYTY5NDQxMzFhMTgzZmIyNjExMzg3MGY2MTIyMg==
+    MmI0ZDFmZWZhYTIxN2FkN2E0MWFmNjk1NDFiYWI3MDgyZmRmZjg2Yw==
   data.tar.gz: !binary |-
-    ZWJjZDEyZjVkM2NmMzhjMzMwN2MwZjAzMDVkN2VkZTUzMGFhMzJiOA==
+    OWRjYjQzZmI5ZGQ4NWYzYTQ1NDk1NDkzYWI2MTZmYTc5NmViZTYwNg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    NWZjNGY4OTk5YmNhMDhlYWNkYWU0ZGIwYzVkNTViMzE1ZTIwN2EzZDdiNDYy
-    YjMwNmEwOTVlYWFiMmM0ZTNkMWQ4MTQwNGM3MDYxZThlMzA1NDJmZjA0NGRj
-    YThiNzMwMDEyOTQ3ZjE0YWJiNDRlNjdiODIwNTk2MjdmMzFlM2E=
+    ZWRkOTNkOTcyNTExN2E1Mzc0NmViNmVhNWNhOWM1NTljNjdhN2QwMTVhOGY4
+    OTU2MTk4YWEzYTE5YWMzODU1N2NjODYzMTY1NTE0ZDE3M2E0NjdiOTA5MDRm
+    M2VjODM1MDc4NzEzNWMwY2ZhZjNhMjM1NTNlNDlhZDE1MmUyMjg=
   data.tar.gz: !binary |-
-    MmJkYWU5NGI2NzhlODcyNzdmMWE1MGIzMzg3YzNjMDQwNTZjNTc3YWI2ZDZk
-    ZWNkNjQ4MzA5M2IwYzYyMWFjNTRmMWFkMTk5Y2UxOGI0NzhkMjFlZDU4NDRh
-    MWY5ZjNmZGEzYzdhMTQ1MDc4Y2EwNjMyMTBiOTAzY2NiOTM4MTk=
+    ODQxZjBkMmNiYTk2ZTY4ZGEwMjUyNjU1NTM2ZGIxM2Q0MmI2NzAzZWY3ZjNl
+    ZTI4MjExNjdjZmEyODcyODczNzQ3NjA3YjFmZGYxZWFiMDBmZDY2YzgzNDQy
+    MTgxZGRlZWUyNjNkNDE1ZjZkOThkYmIwYTYwNzM5ZDBkZDkwZTA=

data/README.md

@@ -4,9 +4,15 @@ Get a new perspective on your data. In-memory [OLAP cubing](http://en.wikipedia.
 
 ![](https://dl.dropboxusercontent.com/u/1133314/i/shades.gif)
 
+## Install
+
+```
+gem install shades
+```
+
 ## As a command line utility for OLAP cubing
 
-The ```shades``` utility will accept whitespace-delimited data, one event per line, preceeded by two commented lines describing the dimensions and data within.
+The ```shades``` utility will accept whitespace-delimited data, one event per line, preceeded by two commented lines describing the dimensions and measures within.
 
 ```
 # dimensions: timestamp transactionid customer item

data/bin/histo

@@ -1,27 +1,84 @@
 #!/usr/bin/env ruby
+#/ Usage: histo [options]
+#/ -p <measure> : Parse shades formatted data and histogram a specific measure given as the last argument
+#/ -n <num>     : the max number of bins to include in the output (default = 10)
+#/ -w <width>   : the width of the histogram bars (default = 30)
+#/ -l           : output ascii art on a log scale (default is linear).
+#/              : tip: if the value distribution looks linear using this option then you're dealing with a power law.
+#/ -h           : show this usage
 $: << File.realpath(File.dirname(__FILE__) + "/../lib")
 
 require 'shades'
+require 'getoptlong'
+require 'pathname'
 
-def main(mkey, max_buckets, output_width)
-  
-  # set up the histogram data to accept streaming input
-  histo = Shades::DynamicHistogram.new(max_buckets)
-  p = Shades::StreamParser.new do |e|
-    histo.add(e.measure(mkey))
-  end
+def usage
+  program = Pathname.new(__FILE__).realpath
+  help = `grep ^\#\/ #{program} | cut -c4-`
+  puts help
+  exit 1
+end
 
-  # stream stdin lines to the parser
-  $stdin.each_line do |line|
-    p << line
-  end
+def main(mkey, max_bins, ascii_width, log_base)
+
+  histo = Shades::DynamicHistogram.new(max_bins)
+
+  if mkey.nil?
+    # just read a stream of numbers from stdin
+    $stdin.each_line do |line|
+      line.scan(/(?:\d+\.?\d*|\d*\.\d+)/).each do |s|
+        begin
+          n = Float(s)
+          histo.add(n)
+        rescue => err
+          puts "error parsing %s as float" % s
+          puts err.message
+        end
+      end
+    end
+  else
+    # set up the histogram data to accept streaming input
+    p = Shades::StreamParser.new do |e|
+      histo.add(e.measure(mkey))
+    end
 
-  $stdout.puts histo.ascii_art
+    # stream stdin lines to the parser
+    $stdin.each_line do |line|
+      p << line
+    end
+  end
+  $stdout.puts histo.ascii_art(ascii_width, log_base)
 end
 
-measure = ARGV[-1]
-max_buckets = 10
-output_width = 30
+opts = GetoptLong.new
+opts.set_options(
+  # parse shades formatted data and histogram a specific measure given as the last argument
+  ["-p", "--parse"       , GetoptLong::OPTIONAL_ARGUMENT],
+  ["-n", "--max-bins"    , GetoptLong::OPTIONAL_ARGUMENT],
+  ["-w", "--ascii-width" , GetoptLong::OPTIONAL_ARGUMENT],
+  ["-l", "--log-base"    , GetoptLong::OPTIONAL_ARGUMENT],
+  ["-h", "--help"        , GetoptLong::NO_ARGUMENT],
+)
+
+measure = nil
+max_bins = 10
+ascii_width = 30
+log_base = 0
+
+opts.each do |opt, arg|
+  case opt
+    when '-h'
+      usage
+    when '-p'
+      measure = arg
+    when '-n'
+      max_bins = arg.to_i
+    when '-w'
+      ascii_width = arg.to_i
+    when '-l'
+      log_base = arg.to_i
+  end
+end
 
-main(measure, max_buckets, output_width)
+main(measure, max_bins, ascii_width, log_base)
 

data/lib/formatter.rb

@@ -4,7 +4,10 @@ module Shades
       @spacer = spacer
     end
     def text(out, events)
-      metadata = events[0].metadata unless events.empty?
+      if events.empty?
+        return
+      end
+      metadata = events[0].metadata
       lines = []
       out.puts "# dimensions: %s" % (metadata.dimensions.join(@spacer))
       out.puts "# measures: %s"   % (metadata.measures.join(@spacer))
@@ -13,7 +16,10 @@ module Shades
       end
     end
     def pretty_text(out, events)
-      metadata = events[0].metadata unless events.empty?
+      if events.empty?
+        return
+      end
+      metadata = events[0].metadata
       lines = []
       (events.length+1).times {|i|lines[i] = []}
       metadata.dimensions.each do |d|

data/lib/histo.rb

@@ -2,7 +2,7 @@
 module Shades
 
   # streaming histograms:
-  # implementation of the clojure library from BigML: https://github.com/bigmlcom/histogram
+  # Ruby port of the Clojure library from BigML: https://github.com/bigmlcom/histogram
   class DynamicHistogram
 
     def initialize(max_size)
@@ -18,8 +18,14 @@ module Shades
       @res.lines
     end
 
-    def ascii_art
-      @res.histo_text
+    def ascii_art(output_width, log_base)
+      @res.histo_text(output_width) do |x|
+        if log_base != 0
+          Math::log(x, log_base)
+        else
+          x
+        end
+      end
     end
   end
 
@@ -90,18 +96,23 @@ module Shades
     ##
     ## So, the line above that reads "0.502 ( 27) ##############################"
     ## can be read as: "There are 27 values close to 0.502"
-    def histo_text
+    def histo_text(output_width)
       a = []
       max_bin_count = 1
-      width = 30
       @bins.each do |b|
         if b.count > max_bin_count
           max_bin_count = b.count
         end
       end
+      scaled_max = yield max_bin_count
+      output_width -= 23
+      if output_width < 10
+        output_width = 10
+      end
       @bins.each do |b|
-        repeat = width * Float(b.count)/Float(max_bin_count)
-        a << "%10.3f (%3d) %s" % [b.mean, b.count, '#' * repeat]
+        scaled_value = yield b.count
+        repeat = output_width * ( scaled_value / scaled_max )
+        a << "%14.3f (%5d) %s" % [b.mean, b.count, '#' * repeat]
       end
       a.join("\n")
     end

data/lib/model.rb

@@ -10,15 +10,21 @@ module Shades
     # parse an event line that adheres to this metadat
     def parse_event(line, sep)
       values = line.split(sep)
-      d = {}
-      @dimensions.zip(values.take(@dimensions.length)).each do |k, v|
-        d[k] = v.strip
+      begin
+        d = {}
+        @dimensions.zip(values.take(@dimensions.length)).each do |k, v|
+          d[k] = v.strip
+        end
+        m = {}
+        @measures.zip(values.drop(@dimensions.length)).each do |k, v|
+          m[k] = Float(v.strip)
+        end
+        return Event.new(self, d, m)
+      rescue => err
+        puts err.message
+        puts "line: #{line}"
       end
-      m = {}
-      @measures.zip(values.drop(@dimensions.length)).each do |k, v|
-        m[k] = Float(v.strip)
-      end
-      Event.new(self, d, m)
+      nil
     end
   end
 

data/lib/queryparser.rb

@@ -7,7 +7,7 @@ module Shades
   class QueryParser
 
     def self.parse(qs)
-      parts = qs.scan(/\w+/)
+      parts = qs.scan(/[\w\.]+/)
       tokens = []
       t = BeginRollupToken.new
       parts.each do |p|

data/lib/streamparser.rb

@@ -12,9 +12,13 @@ module Shades
       line.strip!
       if !@metadata.nil?
         event = @metadata.parse_event(line, /\s+/)
-        @receiver.call(event)
+        if event.nil?
+          puts line
+        else
+          @receiver.call(event)
+        end
       elsif line.start_with?("#")
-        parts = line.scan(/\w+/)
+        parts = line.scan(/[\w\.]+/)
         if parts[0].eql?("dimensions")
           @dimensions = parts.drop(1)
         elsif parts[0].eql?("measures")

data/shades.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'shades'
-  s.version = '0.11'
+  s.version = '0.12'
 
   s.summary = "Get a new perspective on your data. In-memory data cubing of event data for Ruby."
   s.description = <<-EOF

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shades
 version: !ruby/object:Gem::Version
-  version: '0.11'
+  version: '0.12'
 platform: ruby
 authors:
 - Dietrich Featherston
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-25 00:00:00.000000000 Z
+date: 2013-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler