ustate-client 0.0.6 → 0.0.7

data/lib/ustate/query_string.treetop

@@ -2,7 +2,7 @@ module UState
   grammar QueryString
     # (state =~ "foo" or state == "bar") and service == "foo"
     # Binding order proceeds from loosest to tightest.
-    
+
     rule or
       first:and rest:(space 'or' space and)* {
         def query
@@ -21,15 +21,15 @@ module UState
     end
 
     rule and
-      first:primary rest:(space 'and' space primary)* {
+      first:(not / primary) rest:(space 'and' space p:(not / primary))* {
         def query
-          rest.elements.map { |x| x.primary }.inject(first.query) do |a, sub|
+          rest.elements.map { |x| x.p }.inject(first.query) do |a, sub|
             Query::And.new a, sub.query
           end
         end
 
         def sql
-          rest.elements.map { |x| x.primary }.
+          rest.elements.map { |x| x.p }.
             inject(first.sql) do |a, sub|
               a & sub.sql
             end
@@ -37,6 +37,18 @@ module UState
       }
     end
 
+    rule not
+      'not' space p:(not / primary) {
+        def query
+          Query::Not.new p.query
+        end
+
+        def sql 
+          ~ p.sql
+        end
+      }
+    end
+
     rule primary
       '(' space? x:or space? ')' {
         def query
@@ -52,7 +64,7 @@ module UState
     end 
 
     rule predicate
-      equals / not_equals / approximately
+      true / false / less_equal / less / greater_equal / greater / equals / not_equals / approximately
     end
 
     rule approximately
@@ -66,7 +78,55 @@ module UState
         end
       }
     end   
+    
+    rule less_equal
+      field space? '<=' space? value {
+        def query
+          Query::LessEqual.new field.sql, value.sql
+        end
 
+        def sql
+          Sequel::SQL::BooleanExpression.new(:<=, field.sql, value.sql)
+        end
+      }
+    end
+
+    rule less
+      field space? '<' space? value {
+        def query
+          Query::Less.new field.sql, value.sql
+        end
+
+        def sql
+          Sequel::SQL::BooleanExpression.new(:<, field.sql, value.sql)
+        end
+      }
+    end
+    
+    rule greater_equal
+      field space? '>=' space? value {
+        def query
+          Query::GreaterEqual.new field.sql, value.sql
+        end
+
+        def sql
+          Sequel::SQL::BooleanExpression.new(:>=, field.sql, value.sql)
+        end
+      }
+    end
+
+    rule greater
+      field space? '>' space? value {
+        def query
+          Query::Greater.new field.sql, value.sql
+        end
+
+        def sql
+          Sequel::SQL::BooleanExpression.new(:>, field.sql, value.sql)
+        end
+      }
+    end
+    
     rule not_equals
       field space? '!=' space? value {
         def query
@@ -92,9 +152,41 @@ module UState
     end
 
     rule value
-      double_quoted_string / float / integer / null
+      true / false / double_quoted_string / float / integer / null
     end
 
+    rule true
+      'true' {
+        def query
+          Query::True.new
+        end
+          
+        def ruby_value
+          true
+        end
+
+        def sql
+          Sequel::TRUE
+        end
+      }
+    end
+    
+    rule false
+      'false' {
+        def query
+          Query::False.new
+        end
+          
+        def ruby_value
+          false
+        end
+        
+        def sql
+          Sequel::FALSE
+        end
+      }
+    end
+        
     rule integer
       '-'? [0-9]+ {
         def ruby_value
@@ -116,7 +208,7 @@ module UState
 
     rule field
 #      [a-zA-Z] [a-zA-Z_0-9]*
-      ("state" / "host" / "service" / "description" / "metric_f") {
+      ("state" / "host" / "service" / "description" / "metric_f" / "time") {
         def sql
           text_value.to_sym
         end

data/lib/ustate/reaper.rb

@@ -0,0 +1,126 @@
+module UState
+  class Reaper
+    # The reaper deletes old states from an index.
+    
+    # By default, states die after
+    DEFAULT = 1000
+    # By default, scans for old states every
+    INTERVAL = 10
+
+    attr_reader :default
+    attr_accessor :index
+    attr_accessor :interval
+    attr_accessor :targets
+
+    def initialize(index, opts = {})
+      @index = index
+      @interval = opts[:interval] || INTERVAL
+      @server = opts[:server]
+      @targets = {}
+      @lock = Mutex.new
+      @compiled_targets = {}
+      self.default = opts[:default] || DEFAULT
+
+      start
+    end
+
+    # Transform targets into disjoint queries, ordered by age.
+    # This is kinda ugly; should do write an AST optimizer with boolean 
+    # minimization if anyone starts using this feature heavily.
+    def compile
+      @lock.synchronize do
+        @compiled_targets = {}
+        
+        ordered = @targets.sort do |a,b|
+          # The states with the longest lifetimes must be excluded from
+          # all the younger checks. Nil = +inf.
+          if b[1].nil?
+            1
+          elsif a[1].nil?
+            -1
+          else
+            b[1] <=> a[1]
+          end
+        end
+        
+        excluded = ordered.inject(nil) do |exclude, pair|
+          # Build up a set of all queries which should last *longer* than us.
+          query, age = pair
+          if exclude
+            if age
+              @compiled_targets["(#{query}) and not (#{exclude})"] = age
+            end
+            "(#{query}) or (#{exclude})"
+          else
+            if age
+              @compiled_targets[query] = age
+            end
+            query
+          end
+        end
+
+        # Add default
+        if @default
+          q = if excluded
+                "not (#{excluded})"
+              else
+                "true"
+              end
+          @compiled_targets[q] = @default
+        end
+      end
+    end
+    
+    def default=(d)
+      @default = d
+      compile
+    end
+
+    # Delete states matching query
+    def delete(query)
+      @index.query(Query.new(string: query)).each do |state|
+        @index.on_state_change(
+          state,
+          State.new(
+            host: state.host,
+            service: state.service,
+            state: 'unknown',
+            description: "ustate has not heard from this service since #{Time.at(state.time)}",
+            time: Time.now.to_i
+          )
+        )
+        @index.delete state
+      end
+    end
+
+    # Reap states matching query after age seconds.
+    #
+    # The reaper is gentle; it will give any state specified to reap() the
+    # maximum possible time to live.
+    def reap(query, age)
+      reap! query, age
+      compile
+    end
+
+    def reap!(query, age)
+      @targets[query] = age
+    end
+
+    def start
+      @runner = Thread.new do
+        loop do
+          begin
+            @lock.synchronize do
+              @compiled_targets.each do |query, age|
+                delete "(#{query}) and time < #{(Time.now - age).to_i}"
+              end
+            end
+          rescue Exception => e
+            @server.log.warn e
+          end
+          sleep @interval
+        end
+      end
+    end
+  end
+end

data/lib/ustate/server.rb

@@ -15,11 +15,13 @@ module UState
     require 'ustate/query_string'
     require 'ustate/query/ast'
     require 'ustate/metric_thread'
+    require 'ustate/reaper'
     require 'logger'
     require 'mtrc'
 
     attr_accessor :backends
     attr_accessor :index
+    attr_accessor :reaper
     attr_writer :aggregator
     attr_writer :emailer
     attr_writer :graphite
@@ -35,6 +37,8 @@ module UState
 
       @index = Index.new :server => self
 
+      @reaper = UState::Reaper.new(@index, server: self)
+
       @log = Logger.new('ustate.log', 4, 134217728)
       @log.level = Logger::INFO  
 

data/lib/ustate/server/index.rb

@@ -10,15 +10,12 @@ module UState
 
     THREADS = 1000
     BUFFER_SIZE = 10
-    # Forget about states after
-    EXPIRY = 1000
 
     # Update metrics every
     INSERT_RATE_INTERVAL = 5
     INSERT_TIMES_INTERVAL = 5
     
     attr_reader :db, :queue
-    attr_accessor :expiry
     attr_accessor :insert_rate_interval
     attr_accessor :insert_times_interval
 
@@ -33,8 +30,6 @@ module UState
       @on_state_once = []
       @on_state = []
 
-      @expiry = opts[:expiry] || EXPIRY
-
       @insert_rate_interval = opts[:insert_rate_interval] || INSERT_RATE_INTERVAL
       @insert_times_interval = opts[:insert_times_interval] || INSERT_TIMES_INTERVAL
       setup_db
@@ -52,6 +47,15 @@ module UState
       @insert_rate << 1
     end
 
+    # Removes a state from the index.
+    #
+    # Right now state is anything which responds to #host and #service.
+    # I'll probably evolve the index to support arbitrary operations on all
+    # states matching a query, but haven't thought out the API.
+    def delete(state)
+      @db[:states].filter(host: state.host, service: state.service).delete
+    end
+
     def thread(s)
       Thread.new do
         process s
@@ -146,36 +150,6 @@ module UState
       end
     end
 
-    # Remove states older than @expiry
-    def reap
-      @db[:states].filter { |s| 
-        s.time < (Time.now - @expiry).to_i 
-      }.each do |row|
-        on_state_change(
-          row_to_state(row),
-          row_to_state(
-            row.merge(
-              state: 'unknown',
-              description: "ustate has not heard from this service since #{Time.at(row[:time])}",
-              metric_f: nil, 
-              time: Time.now.to_i
-            )
-          )
-        )
-        @db[:states].filter(host: row[:host], state: row[:state]).delete
-      end
-    end
-
-    # Periodically expire old states.
-    def reaper
-      Thread.new do
-        loop do
-          sleep 1
-          reap
-        end
-      end
-    end
-
     # Converts a row to a State
     def row_to_state(row)
       State.new(row)
@@ -197,7 +171,6 @@ module UState
     def start
       stop!
       @pool = []
-      @reaper = reaper
 
       @insert_rate = MetricThread.new(Mtrc::Rate) do |r|
         self << State.new(

data/lib/ustate/version.rb

@@ -1,3 +1,3 @@
 module UState
-  VERSION = '0.0.6'
+  VERSION = '0.0.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ustate-client
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-20 00:00:00.000000000Z
+date: 2011-10-05 00:00:00.000000000 -07:00
+default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beefcake
-  requirement: &76033910 !ruby/object:Gem::Requirement
+  requirement: &17908120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +22,10 @@ dependencies:
         version: 0.3.5
   type: :runtime
   prerelease: false
-  version_requirements: *76033910
+  version_requirements: *17908120
 - !ruby/object:Gem::Dependency
   name: trollop
-  requirement: &76033510 !ruby/object:Gem::Requirement
+  requirement: &17907540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +33,10 @@ dependencies:
         version: 1.16.2
   type: :runtime
   prerelease: false
-  version_requirements: *76033510
+  version_requirements: *17907540
 - !ruby/object:Gem::Dependency
   name: mtrc
-  requirement: &76033170 !ruby/object:Gem::Requirement
+  requirement: &17907040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +44,7 @@ dependencies:
         version: 0.0.4
   type: :runtime
   prerelease: false
-  version_requirements: *76033170
+  version_requirements: *17907040
 description: 
 email: aphyr@aphyr.com
 executables: []
@@ -51,44 +52,53 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/ustate.rb
-- lib/ustate/graphite.rb
+- lib/ustate/query/not_equals.rb
+- lib/ustate/query/equals.rb
+- lib/ustate/query/false.rb
+- lib/ustate/query/node.rb
+- lib/ustate/query/true.rb
+- lib/ustate/query/or.rb
+- lib/ustate/query/approximately.rb
+- lib/ustate/query/greater.rb
+- lib/ustate/query/and.rb
+- lib/ustate/query/less.rb
+- lib/ustate/query/ast.rb
+- lib/ustate/query/less_equal.rb
+- lib/ustate/query/greater_equal.rb
+- lib/ustate/query/not.rb
 - lib/ustate/query_string.treetop
-- lib/ustate/message.rb
+- lib/ustate/server.rb
 - lib/ustate/auto_state.rb
-- lib/ustate/metric_thread.rb
-- lib/ustate/aggregator.rb
+- lib/ustate/emailer.rb
+- lib/ustate/query.rb
 - lib/ustate/version.rb
-- lib/ustate/server/graphite.rb
+- lib/ustate/aggregator.rb
+- lib/ustate/state.rb
+- lib/ustate/reaper.rb
+- lib/ustate/graphite.rb
+- lib/ustate/client/query.rb
+- lib/ustate/metric_thread.rb
+- lib/ustate/query_string.rb
+- lib/ustate/server/index.rb
 - lib/ustate/server/backends.rb
-- lib/ustate/server/backends/base.rb
 - lib/ustate/server/backends/tcp.rb
-- lib/ustate/server/index.rb
+- lib/ustate/server/backends/base.rb
+- lib/ustate/server/graphite.rb
 - lib/ustate/server/connection.rb
-- lib/ustate/server.rb
-- lib/ustate/dash.rb
-- lib/ustate/state.rb
-- lib/ustate/query.rb
-- lib/ustate/client/query.rb
 - lib/ustate/client.rb
-- lib/ustate/dash/rack/static.rb
+- lib/ustate/message.rb
+- lib/ustate/dash/helper/renderer.rb
+- lib/ustate/dash/views/css.scss
 - lib/ustate/dash/views/index.erubis
 - lib/ustate/dash/views/layout.erubis
-- lib/ustate/dash/views/css.scss
+- lib/ustate/dash/rack/static.rb
 - lib/ustate/dash/controller/index.rb
 - lib/ustate/dash/controller/css.rb
 - lib/ustate/dash/config.rb
-- lib/ustate/dash/helper/renderer.rb
-- lib/ustate/query/or.rb
-- lib/ustate/query/not_equals.rb
-- lib/ustate/query/ast.rb
-- lib/ustate/query/equals.rb
-- lib/ustate/query/approximately.rb
-- lib/ustate/query/not.rb
-- lib/ustate/query/and.rb
-- lib/ustate/query_string.rb
-- lib/ustate/emailer.rb
+- lib/ustate/dash.rb
 - LICENSE
 - README.markdown
+has_rdoc: true
 homepage: https://github.com/aphyr/ustate
 licenses: []
 post_install_message: 
@@ -109,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: ustate-client
-rubygems_version: 1.8.10
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Client for the distributed state server ustate.