shades 0.15 → 0.16

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MDdiMzYxZDQzNDdmOTU2NDI5MTU2YTliYmQ0NDNiNjdjN2ExOTBkOA==
+    MzlmYmQ3MTc0MTQ5NWM3MjNlMDBhNzU5MGIzMDM2ODk4NmQ0OTcyOQ==
   data.tar.gz: !binary |-
-    MGY3NTM1OGI3MDZiMTE0NzcwNmMwYzQ5MjAyYzdjYzE4M2IwMGZjOA==
+    OWM3ZjJjNDE5NzI4OGU0MWM3NmRhNzAzOWI5NGVjMmNjYTQ3ZmEwOQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    ZGY5MzNmYTM2NTdkZjRjMTViNzA0MmFlN2NjNjAwM2JiNGY0ZWU0ZmE1YWMz
-    ZTg0NTViZTJmY2I0ZTIyOGFmNjk0N2Y5YjcwZDQ3N2RhZjNjMDlmNjBiOGYy
-    YWRjZDA4YzZjMGQyYmRhMzkwOTA3ZTU5NmMzM2VjMjVmYWU3OTM=
+    MGVkM2YxOWEwNzgyMWM2NzBlNTViOGY0ZjQ5YjEyMDg2OTY2ZWI0YmFjNWI3
+    YTNiOTM2OGI1YjllNmQ3YzM0NjM4ZTA0NDIzYjQzOGI3NDU2ZDQzODdmOWI3
+    M2IwMjUxMzJkNmRjODkzMjFmZTk5ODRmNWUwZTYyNzVjNzFmNjA=
   data.tar.gz: !binary |-
-    YmU2YTc2YzcwZDMyOGM0NmRkOTA4M2E3MzgwODhiNWFiYjNmMjRkNjJjNDYy
-    NWUxMzI1YjkyN2JlYmZkOGYxZjFmOGE1NTc5OTExZWNiNmM5ZjM0ZTJjYjMw
-    MDAzMWEzNWVmZDVkYmVjMGJlYzE3ZTIwNTY5ZDlhZDlhOTFmY2Y=
+    ZTc0YzY2NzA1MDgzZGFkNGVhNTY4Njc0ODBmYjY3MzczOTQzN2RhZTdlNjJj
+    ZTZhMDZlMmZjMWNlYTJiZWQ4ODY4YjczYjFmZTUwOGM2NmIzZjBmNWI4ZDQw
+    NjI0NmZhMGM1OGI1Nzg4ZDIyZjUyZTIwNTI2YmViY2Q0YjllY2M=

data/.gitignore

@@ -11,6 +11,7 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+Gemfile.lock
 
 # YARD artifacts
 .yardoc

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Rakefile

@@ -1 +1,5 @@
 require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec

data/lib/shades.rb

@@ -1,10 +1,10 @@
-require 'model'
-require 'cube'
-require 'histo'
-require 'stats'
-require 'streamparser'
-require 'queryparser'
-require 'formatter'
+require 'shades/model'
+require 'shades/cube'
+require 'shades/histo'
+require 'shades/stats'
+require 'shades/streamparser'
+require 'shades/queryparser'
+require 'shades/formatter'
 
 module Shades
 end

data/lib/shades/query.treetop

@@ -0,0 +1,34 @@
+module Shades
+  grammar QueryGrammar
+    rule query
+      space* rollup_list
+        space* 'by' space* categorization_list:identifier_list
+        optional_sorting:( space* 'order by' space* sorting_list:identifier_list )?
+        space* <QueryNode>
+    end
+
+    rule space
+      [\s]+
+    end
+
+    rule rollup_list
+      rollup rest_rollups:( ',' space* rollup )* <RollupListNode>
+    end
+
+    rule rollup
+      stat_type '(' measures:identifier_list ')' <RollupNode>
+    end
+
+    rule stat_type
+      [\w]+ <StatTypeNode>
+    end
+
+    rule identifier_list
+      identifier rest_identifiers:( ',' space* identifier )* <IdentifierListNode>
+    end
+
+    rule identifier
+      [\w]+ <IdentifierNode>
+    end
+  end
+end

data/lib/shades/queryparser.rb

@@ -0,0 +1,96 @@
+require "treetop"
+
+module Shades
+  module QueryGrammar
+    class QueryNode < Treetop::Runtime::SyntaxNode
+      def rollup_nodes
+        rollup_list.to_a
+      end
+
+      def categorization_nodes
+        categorization_list.to_a
+      end
+
+      def sorting_nodes
+        if optional_sorting.respond_to?(:sorting_list)
+          optional_sorting.sorting_list.to_a
+        else
+          []
+        end
+      end
+    end
+
+    class RollupListNode < Treetop::Runtime::SyntaxNode
+      def to_a
+        [rollup] + rest_rollups.elements.map do |comma_and_rollup|
+          comma_and_rollup.rollup
+        end
+      end
+    end
+
+    class IdentifierListNode < Treetop::Runtime::SyntaxNode
+      def to_a
+        [identifier] + rest_identifiers.elements.map do |comma_and_identifier|
+          comma_and_identifier.identifier
+        end
+      end
+    end
+
+    class RollupNode < Treetop::Runtime::SyntaxNode
+      def stat_type_name
+        stat_type.text_value
+      end
+
+      def measure_names
+        measures.to_a.map(&:text_value)
+      end
+    end
+
+    class StatTypeNode < Treetop::Runtime::SyntaxNode
+    end
+
+    class IdentifierNode < Treetop::Runtime::SyntaxNode
+      def name
+        text_value
+      end
+    end
+  end
+
+  class QueryParser
+    def self.parse(qs, query_factory = Query)
+      @parser_class ||= Treetop.load(File.join(File.dirname(__FILE__), "query.treetop"))
+
+      parser = @parser_class.new
+      unless query_node = parser.parse(qs)
+        raise ArgumentError, "Cannot parse query at character #{parser.index}"
+      end
+
+      query_factory.new(
+        :rollups         => rollups_from_nodes(query_node.rollup_nodes),
+        :categorizations => categorizations_from_nodes(query_node.categorization_nodes),
+        :sorting         => sorting_from_nodes(query_node.sorting_nodes)
+      )
+    end
+
+    def self.rollups_from_nodes(nodes)
+      nodes.flat_map { |node|
+        stat = Stats::StatsType.get(node.stat_type_name)
+        node.measure_names.map { |measure_name|
+          { :stat => stat, :measure => measure_name }
+        }
+      }
+    end
+
+    def self.categorizations_from_nodes(nodes)
+      nodes.map { |categorization|
+        categorization.name
+      }
+    end
+
+    def self.sorting_from_nodes(nodes)
+      nodes.map { |sorting|
+        { :key => sorting.name, :asc => true }
+      }
+    end
+  end
+end

data/lib/shades/shades.rb

@@ -0,0 +1,10 @@
+require 'shades/model'
+require 'shades/cube'
+require 'shades/histo'
+require 'shades/stats'
+require 'shades/streamparser'
+require 'shades/queryparser'
+require 'shades/formatter'
+
+module Shades
+end

data/shades.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'shades'
-  s.version = '0.15'
+  s.version = '0.16'
 
   s.summary = "Get a new perspective on your data. In-memory data cubing of event data for Ruby."
   s.description = <<-EOF
@@ -12,6 +12,8 @@ Gem::Specification.new do |s|
   s.require_paths = %w[lib]
   s.executables = ["shades", "histo"]
 
+  s.add_dependency 'treetop', '~> 1.4.14'
+
   s.add_development_dependency 'rake-compiler'
   s.add_development_dependency 'rspec'
   s.add_development_dependency 'rdoc'

data/spec/queryparser_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+require 'ostruct'
+
+module Shades
+  describe QueryParser do
+    it "parses rollups out of the query expression" do
+      query = QueryParser.parse("sum(amount) by transactionid", OpenStruct)
+
+      expect(query.rollups.length).to eq(1)
+      expect(query.rollups[0][:measure]).to eq("amount")
+      expect(query.rollups[0][:stat]).to eq(Stats::SUM)
+    end
+
+    it "parses multiple rollups out of the query expression" do
+      query = QueryParser.parse("sum(amount), max(quantity) by transactionid", OpenStruct)
+
+      expect(query.rollups.length).to eq(2)
+      expect(query.rollups[0][:measure]).to eq("amount")
+      expect(query.rollups[1][:measure]).to eq("quantity")
+    end
+
+    it "parses categorizations out of the query expression" do
+      query = QueryParser.parse("sum(amount) by transactionid", OpenStruct)
+      expect(query.categorizations).to eq(["transactionid"])
+    end
+
+    it "parses multiple categorizations out of the query expression" do
+      query = QueryParser.parse("sum(amount) by customer, item", OpenStruct)
+      expect(query.categorizations).to eq(["customer", "item"])
+    end
+
+    it "parses sort descriptions out of the query expression" do
+      query = QueryParser.parse("sum(amount) by transactionid order by amount", OpenStruct)
+
+      expect(query.sorting.length).to eq(1)
+      expect(query.sorting[0][:key]).to eq("amount")
+      expect(query.sorting[0][:asc]).to eq(true)
+    end
+
+    it "parses multiple sort descriptions out of the query expression" do
+      query = QueryParser.parse("sum(amount) by transactionid order by amount, customerid", OpenStruct)
+
+      expect(query.sorting.length).to eq(2)
+      expect(query.sorting[0][:key]).to eq("amount")
+      expect(query.sorting[1][:key]).to eq("customerid")
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'shades'
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  config.order = 'random'
+end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: shades
 version: !ruby/object:Gem::Version
-  version: '0.15'
+  version: '0.16'
 platform: ruby
 authors:
 - Dietrich Featherston
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-08 00:00:00.000000000 Z
+date: 2013-08-26 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: treetop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.14
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.14
 - !ruby/object:Gem::Dependency
   name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
@@ -63,31 +77,38 @@ executables:
 extensions: []
 extra_rdoc_files:
 - README.md
-- lib/cube.rb
-- lib/formatter.rb
-- lib/histo.rb
-- lib/model.rb
-- lib/queryparser.rb
+- lib/shades/cube.rb
+- lib/shades/formatter.rb
+- lib/shades/histo.rb
+- lib/shades/model.rb
+- lib/shades/queryparser.rb
+- lib/shades/shades.rb
+- lib/shades/stats.rb
+- lib/shades/streamparser.rb
 - lib/shades.rb
-- lib/stats.rb
-- lib/streamparser.rb
 files:
 - .gitignore
+- .rspec
+- Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - bin/histo
 - bin/shades
-- lib/cube.rb
-- lib/formatter.rb
-- lib/histo.rb
-- lib/model.rb
-- lib/queryparser.rb
 - lib/shades.rb
-- lib/stats.rb
-- lib/streamparser.rb
+- lib/shades/cube.rb
+- lib/shades/formatter.rb
+- lib/shades/histo.rb
+- lib/shades/model.rb
+- lib/shades/query.treetop
+- lib/shades/queryparser.rb
+- lib/shades/shades.rb
+- lib/shades/stats.rb
+- lib/shades/streamparser.rb
 - script/build-ctags
 - shades.gemspec
+- spec/queryparser_spec.rb
+- spec/spec_helper.rb
 - transactions.txt
 homepage: https://github.com/d2fn/shades-rb
 licenses:
@@ -115,7 +136,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: shades
-rubygems_version: 2.0.3
+rubygems_version: 2.0.7
 signing_key: 
 specification_version: 4
 summary: Get a new perspective on your data. In-memory data cubing of event data for

data/lib/queryparser.rb

@@ -1,167 +0,0 @@
-module Shades
-
-  ## queries are of the form:
-  ## <stat-type> <measure>[, [<stat-type>] <measure>]* by <dimension>[, <dimension>] order by <dimension|measure>[, <dimension|measure>]*
-  ## for example, to get the mean load1, and load5 measures by unique combination of host role and kernel version:
-  ##   mean load1, load5 by role, kernelversion
-  class QueryParser
-
-    def self.parse(qs)
-      parts = qs.scan(/[\w\.]+/)
-      tokens = []
-      t = BeginRollupToken.new
-      parts.each do |p|
-        t = t.emit(p)
-        tokens << t
-      end
-      rollups = rollups_pass(tokens)
-      categorizations = categorizations_pass(tokens)
-      sorting = sorting_pass(tokens)
-      Query.new(:categorizations => categorizations, :rollups => rollups, :sorting => sorting)
-    end
-
-    def self.rollups_pass(tokens)
-      stat = nil
-      r = []
-      tokens.each do |t|
-        if t.kind_of? StatTypeToken
-          stat = t.stat
-        elsif t.kind_of? MeasureRefToken
-          r << { :measure => t.text, :stat => stat }
-        end
-      end
-      r
-    end
-
-    def self.categorizations_pass(tokens)
-      d = []
-      tokens.each do |t|
-        if t.kind_of? DimensionRefToken
-          d << t.text
-        end
-      end
-      d
-    end
-
-    def self.sorting_pass(tokens)
-      s = []
-      tokens.each do |t|
-        if t.kind_of? SortKeyToken
-          s << { :key => t.text, :asc => true }
-        end
-      end
-      s
-    end
-  end
-
-  class Token
-    attr_accessor :text
-    def initialize(s)
-      @text = s
-    end
-    def to_s
-      @text
-    end
-  end
-
-  class BeginRollupToken < Token
-    def initialize
-      super("<begin>")
-    end
-    def emit(s)
-      StatTypeToken::parse(s)
-    end
-  end
-
-  class StatTypeToken < Token
-    attr_accessor :stat
-    def initialize(name, stat)
-      super(name)
-      @stat = stat
-    end
-
-    ## context free parsing of the next token to be called by the prior token
-    def self.parse(s)
-      stat = Stats::StatsType::get(s)
-      if stat.nil?
-        nil
-      else
-        StatTypeToken.new(s, stat)
-      end
-    end
-
-    ## given the next string, parse and return the next token
-    def emit(s)
-      # a measure must always follow a stat
-      MeasureRefToken::parse(s)
-    end
-  end
-
-  class MeasureRefToken < Token
-    def self.parse(s)
-      MeasureRefToken.new(s)
-    end
-    def emit(s)
-      if s.downcase.eql?("by")
-        BeginCategorizationToken::parse(s)
-      else
-        t = StatTypeToken::parse(s)
-        if !t.nil?
-          t
-        else
-          MeasureRefToken::parse(s)
-        end
-      end
-    end
-  end
-
-  class BeginCategorizationToken < Token
-    def self.parse(s)
-      BeginCategorizationToken.new(s)
-    end
-    def emit(s)
-      DimensionRefToken::parse(s)
-    end
-  end
-
-  class DimensionRefToken < Token
-    def self.parse(s)
-      DimensionRefToken.new(s)
-    end
-    def emit(s)
-      if s.downcase.eql?("order")
-        OrderToken::parse(s)
-      else
-        DimensionRefToken::parse(s)
-      end
-    end
-  end
-
-  class OrderToken < Token
-    def self.parse(s)
-      OrderToken.new(s)
-    end
-    def emit(s)
-      # by
-      BeginSortingToken::parse(s)
-    end
-  end
-
-  class BeginSortingToken < Token
-    def self.parse(s)
-      BeginSortingToken.new(s)
-    end
-    def emit(s)
-      SortKeyToken::parse(s)
-    end
-  end
-
-  class SortKeyToken < Token
-    def self.parse(s)
-      SortKeyToken.new(s)
-    end
-    def emit(s)
-      SortKeyToken::parse(s)
-    end
-  end
-end