tupelo 0.21 → 0.22

data/example/riemann/v1/riemann.rb

@@ -1,6 +1,6 @@
 # A toy implementation of Riemann (http://riemann.io).
 #
-# Version 1 uses the default tuplespace for all subspaces, which is inefficient
+# Version 1 uses the default tuplestore for all subspaces, which is inefficient
 # for searching.
 
 require 'tupelo/app'
@@ -28,7 +28,7 @@ Tupelo.application do
     child subscribe: "event", passive: true do
       log.progname = "consumer #{i}"
       read subspace("event") do |event|
-        log event ### need filtering, actions, etc.
+        log event # add analytics here
       end
     end
   end

data/example/riemann/v2/event-template.rb

@@ -0,0 +1,71 @@
+# Hard-coded to work with tuples belonging to the "event" subspace 
+# and with the SqliteEventStore table defined. This template is designed
+# for range queries on service, service and host, or service, host, and time,
+# using the composite index.
+class EventTemplate
+  # Template defining the whole event subspace, of which this instance will
+  # match a subset.
+  attr_reader :event_template
+
+  attr_reader :service, :host, :time
+
+  ## todo: support queries by tag or custom key
+
+  # Service, host, and time can be intervals or single values or nil to match
+  # any value. The event_template must be a template that matches all tuples
+  # in the event subspace, such as subspace("event"). The EventTemplate will
+  # match a subset of this subspace.
+  def initialize event_template, service: nil, host: nil, time: nil
+    @event_template = event_template
+    @service = service
+    @host = host
+    @time = time
+  end
+
+  # We only need to define this method if we plan to wait for event tuples
+  # locally using this template, i.e. read(template) or take(template).
+  # Non-waiting queries (such as read_all) just use #find_in or #find_all_in.
+  def === tuple
+    @event_template === tuple and
+    !@service || @service === tuple[:service] and
+    !@host || @host === tuple[:host] and
+    !@time || @time === tuple[:time]
+  end
+  
+  # Returns a dataset corresponding to this particular template.
+  # Dataset has all columns, including id, because we need id to
+  # populate tags and custom key-value data, if any.
+  def dataset store
+    where_terms = {}
+    where_terms[:service] = @service if @service
+    where_terms[:host] = @host if @host
+    where_terms[:time] = @time if @time
+    store.events.where(where_terms)
+  end
+
+  # Optimized search function to find a template match that exists already in
+  # the table. For operations that wait for a match, #=== is used instead.
+  def find_in store, distinct_from: []
+    matches = dataset(store).limit(distinct_from.size + 1).all
+
+    distinct_from.each do |tuple|
+      if i=matches.index(tuple)
+        matches.delete_at i
+      end
+    end
+
+    store.repopulate(matches.first)
+  end
+
+  def find_all_in store
+    if block_given?
+      dataset(store).each do |tuple|
+        yield store.repopulate(tuple)
+      end
+    else
+      dataset(store).map do |tuple|
+        store.repopulate(tuple)
+      end
+    end
+  end
+end

data/example/riemann/v2/expirer.rb

@@ -16,7 +16,7 @@ class Tupelo::Client
     loop do
       event = read subspace("event") # event with lowest time+ttl
         # This just happens to be true, but alternately we could define
-        # OrderedEventTemplate.frist() to pass in a template that explicitly
+        # OrderedEventTemplate.first() to make a template that explicitly
         # finds the lowest key.
       dt_next_expiration = event["time"] + event["ttl"] - Time.now.to_f
       begin

data/example/riemann/v2/hash-store.rb

@@ -4,6 +4,7 @@ require 'tupelo/archiver/tuplestore'
 # rather than matching with templates or other queries.
 # See also example/multitier/kvstore.rb.
 class HashStore < Tupelo::Archiver::TupleStore
+  # Same as parent, but default the zero_tolerance to 1000.
   def initialize zero_tolerance: 1000
     super
   end

data/example/riemann/v2/ordered-event-store.rb

@@ -2,6 +2,9 @@ require 'rbtree'
 
 # Hard-coded to work with tuples belonging to the "event" subspace defined in
 # event-subspace.rb and with the OrderedEventStore data structure defined below.
+# This class only exposes the "find one item before" functionality of rbtree,
+# and not range searches etc., because this is all that is needed for the
+# expirer.
 class OrderedEventTemplate
   attr_reader :expiration
 
@@ -46,7 +49,7 @@ class OrderedEventStore
 
   attr_reader :tree, :metas
 
-  def initialize
+  def initialize client: nil
     clear
   end
 

data/example/riemann/v2/riemann.rb

@@ -3,12 +3,17 @@
 # Version 2 stores the event subspace using different data
 # structures in different clients, depending on needs:
 #
-#   * generic consumers need to index by host and service
+#   * generic consumers need to index by service, host, and time,
+#     by tags, and by custom keys, so they use in-memory sqlite with
+#     normalized tables.
 #
-#   * the expiration manager need to sort by expiration time
+#   * the expiration manager needs to sort by expiration time, so
+#     it uses an rbtree (an in-memory Red-Black binary tree),
+#     for a good balance of insert and lookup performance.
 #
 #   * the critical event alerter doesn't need to sort or search at all,
-#     it just needs efficient insert and delete. 
+#     it just needs efficient insert and delete, so it uses a store
+#     based on a simple hash table.
 #
 # Run with --http to expose a web API and run a test web client.
 #
@@ -31,12 +36,12 @@ require_relative 'hash-store'
 require_relative 'sqlite-event-store'
 
 N_PRODUCERS = 3
-N_CONSUMERS = 2
+N_CONSUMERS = 1
 
 Tupelo.application do
   local do
     define_event_subspace
-    EVENT_SPACE = client.subspace("event")
+    EVENT_SPACE = subspace("event")
   end
 
   if USE_HTTP
@@ -56,11 +61,13 @@ Tupelo.application do
 
   N_CONSUMERS.times do |i|
     # stores events indexed by host, service
-    child tuplestore: [SqliteEventStore, EVENT_SPACE],
-          subscribe: "event", passive: true do
+    child tuplestore: [SqliteEventStore, EVENT_SPACE.spec],
+          subscribe: "event",
+          symbolize_keys: true, # for ease of use with sequel DB interface
+          passive: true do
       log.progname = "consumer #{i}"
       read subspace("event") do |event|
-        log.info event ### need filtering, actions, etc.
+        log.info event # add analytics here
       end
     end
   end

data/example/riemann/v2/sqlite-event-store.rb

@@ -1,49 +1,5 @@
 require 'sequel'
-
-# Hard-coded to work with tuples belonging to the "event" subspace 
-# and with the SqliteEventStore table defined below. This template is designed
-# for range queries on host, or on host and service, using the composite index.
-class EventTemplate
-  attr_reader :host, :service, :space
-
-  # host and service can be intervals or single values or nil to match any value
-  def initialize host: nil, service: nil, space: nil
-    @host = host
-    @service = service
-    @space = space
-  end
-
-  # we only need to define this method if we plan to wait for event tuples
-  # locally using this template, i.e. read(template) or take(template).
-  # Non-waiting queries (such as read_all) just use #find_in.
-  def === tuple
-    @space === tuple and
-    !@host || @host === tuple["host"] and
-    !@service || @service === tuple["service"]
-  end
-
-  # Optimized search function to find a template match that exists already in
-  # the table. For operations that wait for a match, #=== is used instead.
-  def find_in events, distinct_from: []
-    where_terms = {}
-    where_terms[:host] = @host if @host
-    where_terms[:service] = @service if @service
-
-    matches = events.
-      ### select all but id
-      where(where_terms).
-      limit(distinct_from.size + 1).all
-
-    distinct_from.each do |tuple|
-      if i=matches.index(tuple)
-        matches.delete_at i
-      end
-    end
-
-    matches.first ## get the tags and customs
-    ### convert sym to string?
-  end
-end
+require_relative 'event-template'
 
 # A tuple store that is optimized for event data. These tuples are stored in an
 # in-memory sqlite database table. Tuples that do not fit this pattern (such
@@ -51,20 +7,33 @@ end
 class SqliteEventStore
   include Enumerable
 
-  attr_reader :events, :metas, :space
+  attr_reader :events, :tags, :customs, :alt_customs, :metas
+  
+  # Template for matching all event tuples.
+  attr_reader :event_template
+
+  # Object with #dump and #load methods used to put the custom hash into
+  # a sqlite string column.
+  attr_reader :blobber
+
+  def initialize spec, client: nil
+    @event_template = client.worker.pot_for(spec)
+    @blobber = client.blobber
+      # calling #pot_for in client means that resulting template
+      # will have keys converted as needed (in the case of this client,
+      # to symbols).
 
-  # space should be client.subspace("event"), but really we only need
-  # `space.pot` the portable object template for deciding membership.
-  def initialize space
-    @space = space
     clear
+
+    # To be more general, we could inspect the spec to determine which keys to
+    # use when creating and querying the table
   end
   
   def clear
     @db = Sequel.sqlite
     @db.create_table :events do
       primary_key   :id # id is not significant to our app
-      text          :host, null: false ## need this ?
+      text          :host, null: false
       text          :service, null: false
       text          :state
       number        :time
@@ -72,7 +41,7 @@ class SqliteEventStore
       number        :metric
       number        :ttl
 
-      index         [:host, :service]
+      index         [:service, :host, :time]
     end
 
     @db.create_table :tags do
@@ -85,31 +54,86 @@ class SqliteEventStore
     @db.create_table :customs do
       foreign_key   :event_id, :events
       text          :key
-      text          :value
+      text          :value_blob
       index         :key
       primary_key   [:event_id, :key]
     end
 
-    @events = @db[:events]
+    @events   = @db[:events]
+    @tags     = @db[:tags]
+    @customs  = @db[:customs]
+
+    @alt_customs = Hash.new {|h,k| h[k] = {}}
+      # Alternate store for any custom that has a non-string key. To be
+      # completely correct, we have to support this because the event space
+      # requires it.
+      #
+      # Contains
+      #   {event_id => {key => value}}
+      # where key (and value, of course) are arbitrary tuple data.
+      # We don't have the key index like in the customs table, but that's
+      # ok since the key isn't a string.
+
     @metas = []
   end
 
-  def each
-    events.each do |row|
-      ## extra queries for tags and custom 
-      # yuck, gotta convert symbol keys to string keys:
-      tuple = row.inject({}) {|h, (k,v)| h[k.to_s] = v; h}
-      yield tuple
-    end
-    metas.each do |tuple|
-      yield tuple
+  def collect_tags event_id
+    tags.
+      select(:tag).
+      where(event_id: event_id).
+      map {|row| row[:tag]}
+  end
+
+  def collect_custom event_id
+    custom = customs.
+      select(:key, :value_blob).
+      where(event_id: event_id).
+      inject({}) {|h, row|
+        h[row[:key].to_sym] = blobber.load(row[:value_blob])
+        h
+      }
+
+    if alt_customs.key?(event_id)
+      custom.merge! alt_customs[event_id]
     end
+
+    custom
+  end
+
+  def repopulate tuple
+    event_id = tuple.delete :id
+    tuple[:tags]    = collect_tags(event_id)
+    tuple[:custom]  = collect_custom(event_id)
+    tuple
+  end
+
+  def each
+    events.each {|tuple| yield repopulate(tuple)}
+    metas.each {|tuple| yield tuple}
   end
 
   def insert tuple
     case tuple
-    when space
-      events << tuple ## minus tags and customs
+    when event_template
+      tuple = tuple.dup
+      tuple_tags = tuple.delete :tags
+      tuple_custom = tuple.delete :custom
+
+      event_id = events.insert(tuple)
+
+      tuple_tags.each do |tag|
+        tags << {event_id: event_id, tag: tag}
+      end
+
+      tuple_custom.each do |key, value|
+        if key.kind_of? Symbol
+          blob = Sequel.blob(blobber.dump(value))
+          customs << {event_id: event_id, key: key.to_s, value_blob: blob}
+        else
+          alt_customs[event_id][key] = value
+        end
+      end
+
     else
       metas << tuple
     end
@@ -117,16 +141,27 @@ class SqliteEventStore
 
   def delete_once tuple
     case tuple
-    when space
-      # yuck, gotta convert string keys to symbol keys:
-      row = tuple.inject({}) {|h, (k,v)| h[k.to_sym] = v; h}
-      id = events.select(:id).where(row).limit(1)
-      count = events.where(id: id).delete
+    when event_template
+      tuple = tuple.dup
+      tuple_tags = tuple.delete :tags
+      tuple_custom = tuple.delete :custom
+
+      event_id = events.select(:id).where(tuple).limit(1)
+      count = events.where(id: event_id).count
 
       if count == 0
         false
       elsif count == 1
-        true
+        if tuple_tags.sort == collect_tags(event_id).sort && ## avoid sort?
+           tuple_custom == collect_custom(event_id)
+          tags.where(event_id: event_id).delete
+          customs.where(event_id: event_id).delete
+          alt_customs.delete(event_id)
+          events.where(id: event_id).delete
+          true
+        else
+          false
+        end
       else
         raise "internal error: primary key, id, was not unique"
       end
@@ -157,7 +192,7 @@ class SqliteEventStore
   def find_match_for template, distinct_from: []
     case template
     when EventTemplate
-      template.find_in events, distinct_from: distinct_from
+      template.find_in self, distinct_from: distinct_from
     else
       ## if template can match subspace
       # Fall back to linear search, same as default tuplestore.
@@ -171,4 +206,14 @@ class SqliteEventStore
       ## end
     end
   end
+
+  def find_all_matches_for template, &bl
+    case template
+    when EventTemplate
+      template.find_all_in self, &bl
+    else
+      # Fall back to linear search using #each and #===.
+      grep template, &bl
+    end
+  end
 end

data/example/sqlite/poi-store.rb

@@ -40,7 +40,7 @@ class PoiStore
   
   def clear
     @db = Sequel.sqlite
-    @db.create_table "poi" do
+    @db.create_table :poi do
       primary_key   :id # id is not significant to our app
       float         :lat, null: false
       float         :lng, null: false

data/example/sqlite/poi-template.rb

@@ -7,10 +7,10 @@ class PoiTemplate
   attr_reader :poi_template
 
   # lat and lng can be intervals or single values or nil to match any value
-  def initialize lat: nil, lng: nil, poi_template: nil
+  def initialize poi_template, lat: nil, lng: nil
+    @poi_template = poi_template
     @lat = lat
     @lng = lng
-    @poi_template = poi_template
   end
 
   # we only need to define this method if we plan to wait for poi tuples